diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 86d9d589d350..e807b4d711eb 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -31,7 +31,7 @@ ] && blas.meta.available, useCuda ? config.cudaSupport, - useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin, + useMetalKit ? stdenv.hostPlatform.isAarch64 && stdenv.hostPlatform.isDarwin, # Increases the runtime closure size by ~700M useMpi ? false, useRocm ? config.rocmSupport, @@ -92,7 +92,7 @@ let cudaBuildInputs = with cudaPackages; [ cuda_cudart - cuda_cccl # + cccl # libcublas ]; @@ -166,7 +166,7 @@ effectiveStdenv.mkDerivation (finalAttrs: { # `xcrun` is used find the path of the Metal compiler, which is varible # and not on $PATH # see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion - __noChroot = effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders; + __noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders; nativeBuildInputs = [ @@ -181,10 +181,10 @@ effectiveStdenv.mkDerivation (finalAttrs: { autoAddDriverRunpath ] ++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ] - ++ optionals (effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ]; + ++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ]; buildInputs = - optionals effectiveStdenv.isDarwin darwinBuildInputs + optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs ++ optionals useCuda cudaBuildInputs ++ optionals useMpi [ mpi ] ++ optionals useRocm rocmBuildInputs @@ -245,7 +245,7 @@ effectiveStdenv.mkDerivation (finalAttrs: { # Configurations that are known to result in build failures. Can be # overridden by importing Nixpkgs with `allowBroken = true`. - broken = (useMetalKit && !effectiveStdenv.isDarwin); + broken = (useMetalKit && !effectiveStdenv.hostPlatform.isDarwin); description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}"; homepage = "https://github.com/ggml-org/llama.cpp/"; diff --git a/.devops/openvino.Dockerfile b/.devops/openvino.Dockerfile index 13301ba287dd..e301aa8f5c97 100644 --- a/.devops/openvino.Dockerfile +++ b/.devops/openvino.Dockerfile @@ -1,5 +1,5 @@ -ARG OPENVINO_VERSION_MAJOR=2026.3.1 -ARG OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d +ARG OPENVINO_VERSION_MAJOR=2026.4 +ARG OPENVINO_VERSION_FULL=2026.4.0.22959.99c81491cc3 ARG UBUNTU_VERSION=24.04 # Intel GPU driver versions. https://github.com/intel/compute-runtime/releases @@ -10,9 +10,9 @@ ARG COMPUTE_RUNTIME_VERSION_FULL=26.31.39395.13-0 ARG IGDGMM_VERSION=22.10.0 # Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases -ARG NPU_DRIVER_VERSION=v1.35.0 -ARG NPU_DRIVER_FULL=v1.35.0.20260722-29947505341 -ARG LIBZE1_VERSION=1.28.2-1~24.04~ppa1 +ARG NPU_DRIVER_VERSION=v1.38.0 +ARG NPU_DRIVER_FULL=v1.38.0.20260910-34487311128 +ARG LIBZE1_VERSION=1.32.0-1~24.04~ppa1 # Optional proxy build arguments ARG http_proxy= @@ -173,7 +173,7 @@ RUN --mount=type=cache,target=/var/cache/intel-npu,sharing=locked \ fi; \ DEB=/var/cache/intel-npu/libze1_${LIBZE1_VERSION}_amd64.deb; \ if [ ! -f "$DEB" ]; then \ - wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260606T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \ + wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260830T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \ fi; \ mkdir /tmp/npu/ && cd /tmp/npu/ && tar -xf "$TGZ" && cp "$DEB" .; \ apt-get update; \ diff --git a/.ecrc b/.ecrc index c68877ec211f..0338e4faa4a2 100644 --- a/.ecrc +++ b/.ecrc @@ -1,5 +1,5 @@ { - "Exclude": ["^\\.gitmodules$", "stb_image\\.h"], + "Exclude": ["^\\.gitmodules$", "stb_image\\.h", "examples/test-cmake/build/", "examples/test-cmake/build-subdir/"], "Disable": { "IndentSize": true } diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 0d246533c951..570e83e778f2 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,4 +1,4 @@ -blank_issues_enabled: true +blank_issues_enabled: false contact_links: - name: Got an idea? url: https://github.com/ggml-org/llama.cpp/discussions/categories/ideas diff --git a/.github/actions/get-tag-name/action.yml b/.github/actions/get-tag-name/action.yml index 7ace23b2a3e7..46acce5828e9 100644 --- a/.github/actions/get-tag-name/action.yml +++ b/.github/actions/get-tag-name/action.yml @@ -14,7 +14,7 @@ runs: run: | BUILD_NUMBER="$(git rev-list --count HEAD)" SHORT_HASH="$(git rev-parse --short=7 HEAD)" - if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then + if [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT else SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-') diff --git a/.github/actions/windows-setup-cuda/action.yml b/.github/actions/windows-setup-cuda/action.yml index 917513b85eac..e67b6321e95a 100644 --- a/.github/actions/windows-setup-cuda/action.yml +++ b/.github/actions/windows-setup-cuda/action.yml @@ -100,36 +100,36 @@ runs: echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 echo "CUDA_PATH_V13_1=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - - name: Install Cuda Toolkit 13.3 - if: ${{ inputs.cuda_version == '13.3' }} + - name: Install Cuda Toolkit 13.4 for x64 + if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'x64' }} shell: pwsh run: | - mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" + mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" choco install unzip -y - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.3.29-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.5.1.27-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.3.29-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.3.27-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.3.27-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.3.3.1-archive.zip" - unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_crt-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_cudart-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvcc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvrtc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libcublas-windows-x86_64-13.5.1.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libnvvm-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvtx-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_profiler_api-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\visual_studio_integration-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cccl-windows-x86_64-13.3.3.3.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - echo "CUDA_PATH_V13_3=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.4.49-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.7.0.27-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.4.49-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.4.49-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.4.49-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip" + unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvrtc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-x86_64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvtx-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_profiler_api-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\visual_studio_integration-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - name: Install Cuda Toolkit 13.4 for ARM64 if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'arm64' }} @@ -137,19 +137,19 @@ runs: run: | mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" choco install unzip -y - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cccl-windows-x86_64-13.3.4.1.2-archive.zip" - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_crt-windows-x86_64-13.4.46-archive.zip" - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_nvcc-windows-x86_64-13.4.46-archive.zip" - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/libnvvm-windows-x86_64-13.4.46-archive.zip" - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_cudart-windows-arm64-13.4.46-archive.zip" - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/libcublas-windows-arm64-13.7.0.10-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.4.49-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.7.0.27-archive.zip" unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.1.2-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.10-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 diff --git a/.github/workflows/build-and-test-snapdragon.yml b/.github/workflows/build-and-test-snapdragon.yml index 3e857d48e39f..296780acb4cc 100644 --- a/.github/workflows/build-and-test-snapdragon.yml +++ b/.github/workflows/build-and-test-snapdragon.yml @@ -29,7 +29,7 @@ concurrency: jobs: android-ndk-snapdragon: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # previously ubuntu-latest container: image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7' defaults: @@ -59,7 +59,7 @@ jobs: path: pkg-snapdragon/llama.cpp linux-iot-snapdragon: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # previously ubuntu-latest container: image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7' defaults: diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml index 96ce85737fe6..90960a7f61e8 100644 --- a/.github/workflows/build-android.yml +++ b/.github/workflows/build-android.yml @@ -33,7 +33,7 @@ env: jobs: default: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # previously ubuntu-latest steps: - name: Clone @@ -49,7 +49,7 @@ jobs: distribution: zulu - name: Setup Android SDK - uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 + uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4 with: log-accepted-android-sdk-licenses: false @@ -59,7 +59,7 @@ jobs: ./gradlew build --no-daemon ndk: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # previously ubuntu-latest container: image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.3' defaults: @@ -93,7 +93,7 @@ jobs: path: pkg-adb/llama.cpp arm64: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # previously ubuntu-latest env: NDK_VERSION: "29.0.14206865" @@ -123,7 +123,7 @@ jobs: distribution: temurin - name: Setup Android SDK - uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 + uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4 with: log-accepted-android-sdk-licenses: false diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml index 4a23ec2d4d36..27512a142ec5 100644 --- a/.github/workflows/build-cache.yml +++ b/.github/workflows/build-cache.yml @@ -41,8 +41,8 @@ jobs: env: # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.3.1" - OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d" + OPENVINO_VERSION_MAJOR: "2026.4" + OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3" steps: - name: Clone @@ -69,8 +69,8 @@ jobs: env: # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.3.1" - OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d" + OPENVINO_VERSION_MAJOR: "2026.4" + OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3" steps: - name: Clone diff --git a/.github/workflows/build-cmake-pkg.yml b/.github/workflows/build-cmake-pkg.yml index c44fba2c6953..e83589282f3f 100644 --- a/.github/workflows/build-cmake-pkg.yml +++ b/.github/workflows/build-cmake-pkg.yml @@ -5,7 +5,7 @@ on: jobs: linux: - runs-on: [self-hosted, Linux] + runs-on: [self-hosted, Linux, CPU] steps: - uses: actions/checkout@v6 with: diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml index 9e92314bc2f2..df2d876d5662 100644 --- a/.github/workflows/build-cpu.yml +++ b/.github/workflows/build-cpu.yml @@ -88,7 +88,7 @@ jobs: run: | export PIP_BREAK_SYSTEM_PACKAGES="1" python3 -m pip install --upgrade pip setuptools - pip3 install ./gguf-py + pip3 install ./gguf-py jinja2==3.1.6 - name: ccache-buckets-restore uses: ./.github/actions/ccache-buckets @@ -124,7 +124,7 @@ jobs: id: cmake_test run: | cd build - ctest -L main --verbose --timeout 900 + ctest -L 'main|python' --verbose --timeout 900 - name: Test llama2c conversion id: llama2c_test @@ -221,7 +221,6 @@ jobs: # 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar # $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe) # cd build - # $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1 # & $sde -future -- ctest -L main -C Release --verbose --timeout 900 - name: ccache-clear diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.yml index da61b3353e97..30029887c3d0 100644 --- a/.github/workflows/build-cuda-ubuntu.yml +++ b/.github/workflows/build-cuda-ubuntu.yml @@ -177,7 +177,8 @@ jobs: id: cmake_build run: | cmake -B build -S . \ - -DGGML_MUSA=ON + -DGGML_MUSA=ON \ + -DMUSA_ARCHITECTURES=21 time cmake --build build --config Release -j $(nproc) - name: ccache-buckets-save diff --git a/.github/workflows/build-cuda-windows.yml b/.github/workflows/build-cuda-windows.yml index 416724ec7081..e08553e6cdfc 100644 --- a/.github/workflows/build-cuda-windows.yml +++ b/.github/workflows/build-cuda-windows.yml @@ -34,7 +34,7 @@ jobs: - cuda: '12.4' arch: x64 defines: '-DGGML_CUDA_CUB_3DOT2=ON' - - cuda: '13.3' + - cuda: '13.4' arch: x64 defines: '' - cuda: '13.4' diff --git a/.github/workflows/build-ibm.yml b/.github/workflows/build-ibm.yml index d2e4f3cdaeb7..355487e97b8d 100644 --- a/.github/workflows/build-ibm.yml +++ b/.github/workflows/build-ibm.yml @@ -34,10 +34,15 @@ env: LLAMA_ARG_LOG_TIMESTAMPS: 1 jobs: - ubuntu-24-s390x: + name: ubuntu-24-s390x (VXE ${{ matrix.vxe }}) runs-on: ubuntu-24.04-s390x + strategy: + fail-fast: false + matrix: + vxe: ["ON", "OFF"] # `-DGGML_VXE=ON/OFF` + steps: - name: Clone id: checkout @@ -77,7 +82,8 @@ jobs: run: | cmake -B build \ -DLLAMA_FATAL_WARNINGS=ON \ - -DGGML_RPC=ON + -DGGML_RPC=ON \ + -DGGML_VXE=${{ matrix.vxe }} time cmake --build build --config Release -j $(nproc) - name: Test diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml index 8879a6af16fa..daa08b1bf949 100644 --- a/.github/workflows/build-openvino.yml +++ b/.github/workflows/build-openvino.yml @@ -33,7 +33,7 @@ env: LLAMA_ARG_LOG_PREFIX: 1 LLAMA_ARG_LOG_TIMESTAMPS: 1 # TODO: fix failing tests on OpenVINO backend - CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state" + CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-save-load-state" jobs: ubuntu-24-openvino: @@ -41,8 +41,8 @@ jobs: env: # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.3.1" - OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d" + OPENVINO_VERSION_MAJOR: "2026.4" + OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3" steps: - name: Clone @@ -96,8 +96,8 @@ jobs: env: # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.3.1" - OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d" + OPENVINO_VERSION_MAJOR: "2026.4" + OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3" steps: - name: Clone diff --git a/.github/workflows/build-riscv.yml b/.github/workflows/build-riscv.yml index 13f2576b9f08..23a64454e9db 100644 --- a/.github/workflows/build-riscv.yml +++ b/.github/workflows/build-riscv.yml @@ -106,83 +106,3 @@ jobs: wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256 - - ubuntu-riscv64-native-sanitizer: - runs-on: ubuntu-24.04-riscv - - continue-on-error: true - - strategy: - matrix: - sanitizer: [ADDRESS, THREAD, UNDEFINED] - build_type: [Debug] - - steps: - - name: Install dependencies - run: | - # Set gcc-14 and g++-14 as the default compilers - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100 - - git lfs install - - - name: GCC version check - run: | - gcc --version - g++ --version - - - name: Clone - id: checkout - uses: actions/checkout@v6 - - # note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation - #- name: ccache - # uses: ggml-org/ccache-action@v1.2.24 - # with: - # key: riscv-ubuntu-native-sanitizer-${{ matrix.sanitizer }}-${{ matrix.build_type }} - # evict-old-files: 1d - # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - - - name: Build - id: cmake_build - if: ${{ matrix.sanitizer != 'THREAD' }} - run: | - cmake -B build \ - -DLLAMA_OPENSSL=OFF \ - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DGGML_OPENMP=ON \ - -DLLAMA_BUILD_EXAMPLES=ON \ - -DLLAMA_BUILD_TOOLS=ON \ - -DLLAMA_BUILD_TESTS=OFF \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \ - -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \ - -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 - - cmake --build build --config ${{ matrix.build_type }} -j $(nproc) - - - name: Build (no OpenMP) - id: cmake_build_no_openmp - if: ${{ matrix.sanitizer == 'THREAD' }} - run: | - cmake -B build \ - -DLLAMA_OPENSSL=OFF \ - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DGGML_OPENMP=OFF \ - -DLLAMA_BUILD_EXAMPLES=ON \ - -DLLAMA_BUILD_TOOLS=ON \ - -DLLAMA_BUILD_TESTS=OFF \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \ - -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \ - -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 - - cmake --build build --config ${{ matrix.build_type }} -j $(nproc) - - - name: Test - id: cmake_test - run: | - cd build - ctest -L main --verbose --timeout 900 diff --git a/.github/workflows/build-sanitize.yml b/.github/workflows/build-sanitize.yml index 189b5c0fe7b7..89fcff71d445 100644 --- a/.github/workflows/build-sanitize.yml +++ b/.github/workflows/build-sanitize.yml @@ -101,8 +101,6 @@ jobs: - name: Test id: cmake_test - # skip run in Debug - very slow - if: ${{ matrix.sanitizer != 'UNDEFINED' }} run: | cd build ctest -L main -E tokenizer --verbose --timeout 900 diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml deleted file mode 100644 index ccfe2a604645..000000000000 --- a/.github/workflows/build-self-hosted.yml +++ /dev/null @@ -1,409 +0,0 @@ -name: CI (self-hosted) - -on: - workflow_dispatch: # allows manual triggering - push: - branches: - - master - paths: [ - '.github/workflows/build-self-hosted.yml', - '**/CMakeLists.txt', - '**/.cmake', - '**/*.h', - '**/*.hpp', - '**/*.c', - '**/*.cpp', - '**/*.cu', - '**/*.cuh', - '**/*.swift', - '**/*.m', - '**/*.metal', - '**/*.comp', - '**/*.glsl', - '**/*.wgsl' - ] - - pull_request: - types: [opened, synchronize, reopened] - paths: [ - '.github/workflows/build-self-hosted.yml', - '**/CMakeLists.txt', - '**/.cmake', - '**/*.h', - '**/*.hpp', - '**/*.c', - '**/*.cpp', - '**/*.cu', - '**/*.cuh', - '**/*.swift', - '**/*.m', - '**/*.metal', - '**/*.comp', - '**/*.glsl', - '**/*.wgsl' - ] - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} - cancel-in-progress: true - -env: - # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) - HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} - GGML_NLOOP: 3 - GGML_N_THREADS: 1 - LLAMA_ARG_LOG_COLORS: 1 - LLAMA_ARG_LOG_PREFIX: 1 - LLAMA_ARG_LOG_TIMESTAMPS: 1 - -jobs: - gpu-cuda: - runs-on: [self-hosted, Linux, NVIDIA] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Test - id: ggml-ci - run: | - nvidia-smi - GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - gpu-rocm: - runs-on: [self-hosted, Linux, AMD] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Test - id: ggml-ci - # HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness - # issue on integrated RDNA3.5 (gfx1151) where batched inference returns - # incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches - # restores correctness. Remove once the underlying ROCm/HIP issue is fixed. - env: - HIP_LAUNCH_BLOCKING: "1" - run: | - rocminfo - GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - gpu-vulkan-nvidia-cm: - runs-on: [self-hosted, Linux, NVIDIA] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Test - id: ggml-ci - run: | - vulkaninfo --summary - GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - gpu-vulkan-nvidia-cm2: - runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Test - id: ggml-ci - run: | - vulkaninfo --summary - GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - gpu-webgpu-nvidia: - runs-on: [self-hosted, Linux, NVIDIA, X64] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Dawn Dependency - id: dawn-depends - run: | - DAWN_VERSION="v20260317.182325" - DAWN_OWNER="google" - DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release" - echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" - curl -L -o artifact.tar.gz \ - "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" - mkdir dawn - tar -xvf artifact.tar.gz -C dawn --strip-components=1 - - - name: Test - id: ggml-ci - run: | - GG_BUILD_WEBGPU=1 \ - GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \ - GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \ - bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - # TODO: provision AMX-compatible machine - #cpu-amx: - # runs-on: [self-hosted, Linux, CPU, AMX] - - # steps: - # - name: Clone - # id: checkout - # uses: actions/checkout@v6 - - # - name: Test - # id: ggml-ci - # run: | - # bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - # TODO: provision AMD GPU machine - # amd-vulkan: - # runs-on: [self-hosted, Linux, AMD] - - # steps: - # - name: Clone - # id: checkout - # uses: actions/checkout@v6 - - # - name: Test - # id: ggml-ci - # run: | - # vulkaninfo --summary - # GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - # TODO: provision AMD GPU machine - # amd-rocm: - # runs-on: [self-hosted, Linux, AMD] - - # steps: - # - name: Clone - # id: checkout - # uses: actions/checkout@v6 - - # - name: Test - # id: ggml-ci - # run: | - # amd-smi static - # GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - gpu-metal: - runs-on: [self-hosted, macOS, ARM64] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Test - id: ggml-ci - run: | - GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - gpu-webgpu-apple: - runs-on: [self-hosted, macOS, ARM64] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Dawn Dependency - id: dawn-depends - run: | - DAWN_VERSION="v20260317.182325" - DAWN_OWNER="google" - DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release" - echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" - curl -L -o artifact.tar.gz \ - "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" - mkdir dawn - tar -xvf artifact.tar.gz -C dawn --strip-components=1 - - - name: Test - id: ggml-ci - run: | - GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \ - bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - gpu-vulkan-apple: - runs-on: [self-hosted, macOS, ARM64] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Test - id: ggml-ci - run: | - vulkaninfo --summary - GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - gpu-vulkan-intel-linux: - runs-on: [self-hosted, Linux, Intel] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Test - id: ggml-ci - run: | - vulkaninfo --summary - GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - gpu-vulkan-intel-windows: - runs-on: [self-hosted, Windows, X64, Intel] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Test - id: ggml-ci - shell: C:\msys64\usr\bin\bash.exe --noprofile --norc -eo pipefail "{0}" - env: - MSYSTEM: UCRT64 - CHERE_INVOKING: 1 - PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }} - run: | - vulkaninfo --summary - # Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create - # a valid python environment for testing - LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp - - gpu-openvino-low-perf: - runs-on: [self-hosted, Linux, Intel, OpenVINO] - - env: - # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.3.1" - OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d" - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Setup OpenVINO Toolkit - uses: ./.github/actions/linux-setup-openvino - with: - path: ./openvino_toolkit - version_major: ${{ env.OPENVINO_VERSION_MAJOR }} - version_full: ${{ env.OPENVINO_VERSION_FULL }} - - - name: Install OpenVINO dependencies - run: | - cd ./openvino_toolkit - chmod +x ./install_dependencies/install_openvino_dependencies.sh - echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh - - - name: Test - id: ggml-ci - run: | - source ./openvino_toolkit/setupvars.sh - GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - cpu-x64-high-perf: - runs-on: [self-hosted, Linux, X64] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Test - id: ggml-ci - run: | - LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - cpu-arm64-high-perf-graviton4: - runs-on: ah-ubuntu_22_04-c8g_8x - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Dependencies - id: depends - run: | - set -euxo pipefail - sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \ - apt-get install -y \ - build-essential \ - python3-venv \ - gpg \ - wget \ - time \ - git-lfs - - git lfs install - - # install the latest cmake - sudo install -d /usr/share/keyrings - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \ - | gpg --dearmor \ - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null - echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \ - | sudo tee /etc/apt/sources.list.d/kitware.list - sudo apt-get update - sudo apt-get install -y cmake - - - name: Test - id: ggml-ci - run: | - LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp - - cpu-arm64-graviton4-kleidiai: - runs-on: ah-ubuntu_22_04-c8g_8x - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v6 - - - name: Dependencies - id: depends - run: | - set -euxo pipefail - sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \ - apt-get install -y \ - build-essential \ - python3-venv \ - gpg \ - wget \ - time \ - git-lfs - - git lfs install - - # install the latest cmake - sudo install -d /usr/share/keyrings - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \ - | gpg --dearmor \ - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null - echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \ - | sudo tee /etc/apt/sources.list.d/kitware.list - sudo apt-get update - sudo apt-get install -y cmake - - - name: Test - id: ggml-ci - run: | - GG_BUILD_KLEIDIAI=1 \ - GG_BUILD_EXTRA_TESTS_0=1 \ - bash ./ci/run.sh ./tmp/results ./tmp/mnt diff --git a/.github/workflows/build-sycl.yml b/.github/workflows/build-sycl.yml index 9ddb894f2730..5424a8607e58 100644 --- a/.github/workflows/build-sycl.yml +++ b/.github/workflows/build-sycl.yml @@ -49,7 +49,7 @@ jobs: env: ONEAPI_ROOT: /opt/intel/oneapi/ ONEAPI_INSTALLER_VERSION: "2025.3.3" - LEVEL_ZERO_VERSION: "1.28.2" + LEVEL_ZERO_VERSION: "1.33.1" LEVEL_ZERO_UBUNTU_VERSION: "u24.04" continue-on-error: true @@ -70,9 +70,10 @@ jobs: shell: bash run: | cd /tmp - wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb - wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb - sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb + # v1.33.x renamed the Debian packages to libze1 / libze-dev + wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze1_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze1.deb + wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb + sudo apt-get install -y ./libze1.deb ./libze-dev.deb - name: ccache uses: ggml-org/ccache-action@v1.2.24 @@ -101,7 +102,11 @@ jobs: -DCMAKE_CXX_COMPILER=icpx \ -DLLAMA_OPENSSL=OFF \ -DGGML_NATIVE=OFF \ - -DGGML_SYCL_F16=${{ matrix.fp16 }} + -DGGML_SYCL_F16=${{ matrix.fp16 }} \ + -DGGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON \ + -DGGML_SYCL_DNN=ON \ + -DCMAKE_CXX_FLAGS="-fsycl-unnamed-lambda" \ + -DCMAKE_EXE_LINKER_FLAGS="-fsycl-unnamed-lambda" time cmake --build build --config Release -j $(nproc) - name: ccache-buckets-save @@ -126,7 +131,7 @@ jobs: env: WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel - LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip + LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI" ONEAPI_INSTALLER_VERSION: "2025.3.3" steps: diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml index 9de52e990a67..21d2a773531f 100644 --- a/.github/workflows/build-vulkan.yml +++ b/.github/workflows/build-vulkan.yml @@ -164,9 +164,7 @@ jobs: export GGML_VK_VISIBLE_DEVICES=0 export GGML_VK_DISABLE_F16=1 export GGML_VK_DISABLE_COOPMAT=1 - # This is using llvmpipe and runs slower than other backends - # test-backend-ops is too slow on llvmpipe, skip it - ctest -L main -E test-backend-ops --verbose --timeout 900 + ctest -L main --verbose --timeout 900 windows: runs-on: windows-2025 diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 81b943df7b65..5a3166ce6885 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -68,7 +68,7 @@ jobs: - name: Fetch emdawnwebgpu run: | - DAWN_TAG="v20260317.182325" + DAWN_TAG="v20260908.214631" EMDAWN_PKG="emdawnwebgpu_pkg-${DAWN_TAG}.zip" echo "Downloading ${EMDAWN_PKG}" curl -L -o emdawn.zip \ diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml index e624e3ba8016..ec582ff274b3 100644 --- a/.github/workflows/build-webgpu.yml +++ b/.github/workflows/build-webgpu.yml @@ -77,10 +77,10 @@ jobs: - name: Dawn Dependency id: dawn-depends run: | - DAWN_VERSION="v20260317.182325" + DAWN_VERSION="v20260908.214631" DAWN_OWNER="google" DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release" echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" curl -L -o artifact.tar.gz \ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" @@ -147,10 +147,10 @@ jobs: id: dawn-depends run: | sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev - DAWN_VERSION="v20260317.182325" + DAWN_VERSION="v20260908.214631" DAWN_OWNER="google" DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release" echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" curl -L -o artifact.tar.gz \ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" @@ -190,6 +190,4 @@ jobs: id: cmake_test run: | cd build - # This is using llvmpipe and runs slower than other backends - # test-backend-ops is too slow on llvmpipe, skip it - ctest -L main -E test-backend-ops --verbose --timeout 900 + ctest -L main --verbose --timeout 900 diff --git a/.github/workflows/check-vendor.yml b/.github/workflows/check-vendor.yml index 015629f380ca..1671ed7b8bd2 100644 --- a/.github/workflows/check-vendor.yml +++ b/.github/workflows/check-vendor.yml @@ -19,7 +19,7 @@ on: jobs: check-vendor: - runs-on: [self-hosted, fast] + runs-on: ubuntu-slim steps: - name: Checkout diff --git a/.github/workflows/ci-self-hosted-cpu.yml b/.github/workflows/ci-self-hosted-cpu.yml new file mode 100644 index 000000000000..18bc9f144205 --- /dev/null +++ b/.github/workflows/ci-self-hosted-cpu.yml @@ -0,0 +1,112 @@ +name: CI (self-hosted CPU backend) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/ci-self-hosted-cpu.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/ci-self-hosted-cpu.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + 'ggml/src/*', + 'ggml/src/ggml-cpu/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + cpu-x64-high-perf: + runs-on: [self-hosted, Linux, X64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + cpu-arm64-high-perf-graviton4: + runs-on: ah-ubuntu_24_04-c8g_8x + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + set -euxo pipefail + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \ + apt-get install -y \ + build-essential \ + python3-venv \ + gpg \ + wget \ + time \ + git-lfs + + git lfs install + + # install the latest cmake + sudo install -d /usr/share/keyrings + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \ + | gpg --dearmor \ + | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null + echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \ + | sudo tee /etc/apt/sources.list.d/kitware.list + sudo apt-get update + sudo apt-get install -y cmake + + - name: Test + id: ggml-ci + run: | + LLAMA_ARG_THREADS=$(nproc) \ + GG_BUILD_HIGH_PERF=1 \ + GG_BUILD_NO_BF16=1 \ + GG_BUILD_EXTRA_TESTS_0=1 \ + bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + # TODO: provision AMX-compatible machine + #cpu-amx: + # runs-on: [self-hosted, Linux, CPU, AMX] + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v6 + + # - name: Test + # id: ggml-ci + # run: | + # bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp diff --git a/.github/workflows/ci-self-hosted-cuda.yml b/.github/workflows/ci-self-hosted-cuda.yml new file mode 100644 index 000000000000..5951b52d3781 --- /dev/null +++ b/.github/workflows/ci-self-hosted-cuda.yml @@ -0,0 +1,124 @@ +name: CI (self-hosted CUDA backend) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/ci-self-hosted-cuda.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.cu', + '**/*.cuh' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/ci-self-hosted-cuda.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + 'ggml/src/*', + 'ggml/src/ggml-cpu/**', + 'ggml/src/ggml-cuda/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + gpu-cuda: + runs-on: "hf-jobs-t4-small:cuda13" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y cmake libssl-dev time unzip wget python3 python3-venv python3-pip + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + restore: false + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + with: + key: self-hosted-gpu-cuda + folder: llama.cpp + hf_bucket: ggml-org/cache + + - name: Test + id: ggml-ci + run: | + nvidia-smi + GG_BUILD_CUDA=1 CUDACXX=/usr/local/cuda/bin/nvcc bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: self-hosted-gpu-cuda + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ggml-org/cache + save: true + + gpu-rocm: + runs-on: [self-hosted, Linux, AMD] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + # HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness + # issue on integrated RDNA3.5 (gfx1151) where batched inference returns + # incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches + # restores correctness. Remove once the underlying ROCm/HIP issue is fixed. + env: + HIP_LAUNCH_BLOCKING: "1" + run: | + rocminfo + GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + # TODO: provision AMD GPU machine + # amd-rocm: + # runs-on: [self-hosted, Linux, AMD] + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v6 + + # - name: Test + # id: ggml-ci + # run: | + # amd-smi static + # GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp diff --git a/.github/workflows/ci-self-hosted-kleidiai.yml b/.github/workflows/ci-self-hosted-kleidiai.yml new file mode 100644 index 000000000000..c955aa0022d4 --- /dev/null +++ b/.github/workflows/ci-self-hosted-kleidiai.yml @@ -0,0 +1,85 @@ +name: CI (self-hosted KleidiAI backend) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/ci-self-hosted-kleidiai.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/ci-self-hosted-kleidiai.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + 'ggml/src/*', + 'ggml/src/ggml-cpu/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + cpu-arm64-graviton4-kleidiai: + runs-on: ah-ubuntu_24_04-c8g_8x + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dependencies + id: depends + run: | + set -euxo pipefail + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \ + apt-get install -y \ + build-essential \ + python3-venv \ + gpg \ + wget \ + time \ + git-lfs + + git lfs install + + # install the latest cmake + sudo install -d /usr/share/keyrings + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \ + | gpg --dearmor \ + | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null + echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \ + | sudo tee /etc/apt/sources.list.d/kitware.list + sudo apt-get update + sudo apt-get install -y cmake + + - name: Test + id: ggml-ci + run: | + LLAMA_ARG_THREADS=$(nproc) \ + GG_BUILD_KLEIDIAI=1 \ + GG_BUILD_EXTRA_TESTS_0=1 \ + GG_BUILD_HIGH_PERF=1 \ + bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp diff --git a/.github/workflows/ci-self-hosted-metal.yml b/.github/workflows/ci-self-hosted-metal.yml new file mode 100644 index 000000000000..7af30e294a93 --- /dev/null +++ b/.github/workflows/ci-self-hosted-metal.yml @@ -0,0 +1,59 @@ +name: CI (self-hosted Metal backend) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/ci-self-hosted-metal.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.swift', + '**/*.m', + '**/*.metal' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/ci-self-hosted-metal.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + 'ggml/src/*', + 'ggml/src/ggml-cpu/**', + 'ggml/src/ggml-metal/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + gpu-metal: + runs-on: [self-hosted, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp diff --git a/.github/workflows/ci-self-hosted-openvino.yml b/.github/workflows/ci-self-hosted-openvino.yml new file mode 100644 index 000000000000..e0947c46e045 --- /dev/null +++ b/.github/workflows/ci-self-hosted-openvino.yml @@ -0,0 +1,75 @@ +name: CI (self-hosted OpenVINO backend) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/ci-self-hosted-openvino.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/ci-self-hosted-openvino.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + 'ggml/src/*', + 'ggml/src/ggml-cpu/**', + 'ggml/src/ggml-openvino/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + gpu-openvino-low-perf: + runs-on: [self-hosted, Linux, Intel, OpenVINO] + + env: + # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.4" + OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Setup OpenVINO Toolkit + uses: ./.github/actions/linux-setup-openvino + with: + path: ./openvino_toolkit + version_major: ${{ env.OPENVINO_VERSION_MAJOR }} + version_full: ${{ env.OPENVINO_VERSION_FULL }} + + - name: Install OpenVINO dependencies + run: | + cd ./openvino_toolkit + chmod +x ./install_dependencies/install_openvino_dependencies.sh + echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh + + - name: Test + id: ggml-ci + run: | + source ./openvino_toolkit/setupvars.sh + GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp diff --git a/.github/workflows/ci-self-hosted-vulkan.yml b/.github/workflows/ci-self-hosted-vulkan.yml new file mode 100644 index 000000000000..ffed4b09b0c8 --- /dev/null +++ b/.github/workflows/ci-self-hosted-vulkan.yml @@ -0,0 +1,201 @@ +name: CI (self-hosted Vulkan backend) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/ci-self-hosted-vulkan.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.comp', + '**/*.glsl' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/ci-self-hosted-vulkan.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + 'ggml/src/*', + 'ggml/src/ggml-cpu/**', + 'ggml/src/ggml-vulkan/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + gpu-vulkan-nvidia-cm: + # runs-on: "hf-jobs-t4-small:ubuntu26_04" + runs-on: [self-hosted, Linux, NVIDIA] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + # - name: Install dependencies + # run: | + # sudo apt update + # sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip + + # - name: ccache + # uses: ggml-org/ccache-action@v1.2.24 + # with: + # restore: false + # save: false + + # - name: ccache-buckets-restore + # uses: ./.github/actions/ccache-buckets + # with: + # key: self-hosted-vulkan-nvidia-cm + # folder: llama.cpp + # hf_bucket: ggml-org/cache + + - name: Test + id: ggml-ci + run: | + vulkaninfo --summary + GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + # - name: ccache-buckets-save + # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + # uses: ./.github/actions/ccache-buckets + # env: + # HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + # with: + # key: self-hosted-vulkan-nvidia-cm + # folder: llama.cpp + # evict-old-files: 1d + # hf_bucket: ggml-org/cache + # save: true + + gpu-vulkan-nvidia-cm2: + # runs-on: "hf-jobs-t4-small:ubuntu26_04" + runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + # - name: Install dependencies + # run: | + # sudo apt update + # sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip + + # - name: ccache + # uses: ggml-org/ccache-action@v1.2.24 + # with: + # restore: false + # save: false + + # - name: ccache-buckets-restore + # uses: ./.github/actions/ccache-buckets + # with: + # key: self-hosted-vulkan-nvidia-cm2 + # folder: llama.cpp + # hf_bucket: ggml-org/cache + + - name: Test + id: ggml-ci + run: | + vulkaninfo --summary + GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + # - name: ccache-buckets-save + # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + # uses: ./.github/actions/ccache-buckets + # env: + # HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + # with: + # key: self-hosted-vulkan-nvidia-cm2 + # folder: llama.cpp + # evict-old-files: 1d + # hf_bucket: ggml-org/cache + # save: true + + gpu-vulkan-apple: + runs-on: [self-hosted, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + vulkaninfo --summary + GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + gpu-vulkan-intel-linux: + runs-on: [self-hosted, Linux, Intel] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Test + id: ggml-ci + run: | + vulkaninfo --summary + GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + gpu-vulkan-intel-windows: + runs-on: [self-hosted, Windows, X64, Intel] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + shell: C:\msys64\usr\bin\bash.exe --noprofile --norc -eo pipefail "{0}" + env: + MSYSTEM: UCRT64 + CHERE_INVOKING: 1 + PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }} + run: | + vulkaninfo --summary + # Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create + # a valid python environment for testing + LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp + + # TODO: provision AMD GPU machine + # amd-vulkan: + # runs-on: [self-hosted, Linux, AMD] + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v6 + + # - name: Test + # id: ggml-ci + # run: | + # vulkaninfo --summary + # GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp diff --git a/.github/workflows/ci-self-hosted-webgpu.yml b/.github/workflows/ci-self-hosted-webgpu.yml new file mode 100644 index 000000000000..a6a36a8ebab3 --- /dev/null +++ b/.github/workflows/ci-self-hosted-webgpu.yml @@ -0,0 +1,130 @@ +name: CI (self-hosted WebGPU backend) + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/ci-self-hosted-webgpu.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + '**/*.h', + '**/*.hpp', + '**/*.c', + '**/*.cpp', + '**/*.wgsl' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/ci-self-hosted-webgpu.yml', + 'ci/run.sh', + '**/CMakeLists.txt', + '**/.cmake', + 'ggml/src/*', + 'ggml/src/ggml-cpu/**', + 'ggml/src/ggml-webgpu/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + gpu-webgpu-nvidia: + runs-on: "hf-jobs-t4-small:ubuntu26_04" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + restore: false + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + with: + key: self-hosted-webgpu-nvidia + folder: llama.cpp + hf_bucket: ggml-org/cache + + - name: Dawn Dependency + id: dawn-depends + run: | + DAWN_VERSION="v20260908.214631" + DAWN_OWNER="google" + DAWN_REPO="dawn" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release" + echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + curl -L -o artifact.tar.gz \ + "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + mkdir dawn + tar -xvf artifact.tar.gz -C dawn --strip-components=1 + + - name: Test + id: ggml-ci + run: | + GG_BUILD_WEBGPU=1 \ + GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \ + GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \ + bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: self-hosted-webgpu-nvidia + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ggml-org/cache + save: true + + gpu-webgpu-apple: + runs-on: [self-hosted, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Dawn Dependency + id: dawn-depends + run: | + DAWN_VERSION="v20260908.214631" + DAWN_OWNER="google" + DAWN_REPO="dawn" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release" + echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + curl -L -o artifact.tar.gz \ + "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" + mkdir dawn + tar -xvf artifact.tar.gz -C dawn --strip-components=1 + + - name: Test + id: ggml-ci + run: | + GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \ + bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml index 50b598b84ddd..c88396c0a7d1 100644 --- a/.github/workflows/code-style.yml +++ b/.github/workflows/code-style.yml @@ -15,7 +15,7 @@ concurrency: jobs: model-naming: - runs-on: [self-hosted, fast] + runs-on: ubuntu-slim steps: - uses: actions/checkout@v6 - name: Check model naming conventions diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 61c05dcac590..5527e73b6f3d 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -11,10 +11,12 @@ on: paths: - .github/workflows/copilot-setup-steps.yml +cache-mode: none + jobs: # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot. copilot-setup-steps: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # Set the permissions to the lowest permissions possible needed for your steps. # Copilot will be given its own token for its operations. @@ -31,8 +33,8 @@ jobs: - name: ccache uses: ggml-org/ccache-action@v1.2.24 with: - key: copilot-setup-steps - evict-old-files: 1d + restore: false + save: false - name: Dependencies id: depends diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1de25b522de8..b2aafc98528c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -90,8 +90,8 @@ jobs: { "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-s390x", "prebuilt_ui": true }, { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" }, - { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, - { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" }, + { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, + { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" }, { "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, { "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" }, diff --git a/.github/workflows/editorconfig.yml b/.github/workflows/editorconfig.yml index 59159cd41444..53f6a0ccfda4 100644 --- a/.github/workflows/editorconfig.yml +++ b/.github/workflows/editorconfig.yml @@ -15,7 +15,7 @@ concurrency: jobs: editorconfig: - runs-on: [self-hosted, fast] + runs-on: ubuntu-slim steps: - uses: actions/checkout@v6 - uses: editorconfig-checker/action-editorconfig-checker@840e866d93b8e032123c23bac69dece044d4d84c # v2.2.0 diff --git a/.github/workflows/fusion.yml b/.github/workflows/fusion.yml new file mode 100644 index 000000000000..7c8596467aab --- /dev/null +++ b/.github/workflows/fusion.yml @@ -0,0 +1,71 @@ +name: Fusion + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/fusion.yml', + 'ggml/**', + 'tests/fusion/**', + 'tests/test-fusion.cpp', + 'tests/test-llama-archs.cpp', + 'src/models/**' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/fusion.yml', + 'ggml/**', + 'tests/fusion/**', + 'tests/test-fusion.cpp', + 'tests/test-llama-archs.cpp', + 'src/models/**' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + # TODO: add jobs for other backends as they adopt the fusion debug API + metal: + runs-on: [self-hosted, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DLLAMA_OPENSSL=OFF \ + -DGGML_SCHED_NO_REALLOC=ON \ + -DGGML_BLAS=OFF \ + -DGGML_METAL=ON + time cmake --build build --config Release --target test-llama-archs -j $(sysctl -n hw.logicalcpu) + time cmake --build build --config Release --target test-fusion -j $(sysctl -n hw.logicalcpu) + + - name: Generate models + id: generate_models + run: | + rm -rf build-ci-models && mkdir -p build-ci-models + ./build/bin/test-llama-archs -o build-ci-models + + - name: Test fusion + id: test_fusion + run: | + ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv diff --git a/.github/workflows/gguf-publish.yml b/.github/workflows/gguf-publish.yml index fb8eab3cdb3b..613562479301 100644 --- a/.github/workflows/gguf-publish.yml +++ b/.github/workflows/gguf-publish.yml @@ -21,7 +21,7 @@ on: jobs: deploy: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # previously ubuntu-latest steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml index 40fc86287090..101ad43f33d4 100644 --- a/.github/workflows/make-release.yml +++ b/.github/workflows/make-release.yml @@ -13,16 +13,27 @@ on: required: true type: boolean default: true + skip_apiabi_check: + description: 'Skip API/ABI compatibility check' + required: false + type: boolean + default: false + apiabi_compare_tag: + description: 'Tag to compare against for API/ABI check (default: latest release)' + required: false + type: string + default: '' env: GH_TOKEN: ${{ github.token }} permissions: contents: write + packages: write jobs: make-release: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # previously ubuntu-latest steps: - name: Checkout @@ -32,12 +43,18 @@ jobs: ref: ${{ inputs.commit != '' && inputs.commit || github.ref_name }} fetch-depth: 0 + - name: Install API/ABI check tools + if: ${{ github.event.inputs.skip_apiabi_check != 'true' }} + run: sudo apt-get install -y abi-compliance-checker abigail-tools + - name: Run release checks id: checks run: bash scripts/make-release-checks.sh ${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }} env: GITHUB_REPOSITORY: ${{ github.repository }} RELEASE_BRANCH: ${{ github.ref_name }} + SKIP_APIABI_CHECK: ${{ github.event.inputs.skip_apiabi_check }} + APIABI_COMPARE_TAG: ${{ github.event.inputs.apiabi_compare_tag }} - name: Create release tag if: ${{ github.event.inputs.dry_run == 'false' }} @@ -113,6 +130,29 @@ jobs: data: await fs.readFileSync('./nightly-tag.txt') }); + - name: Re-tag container images with release version + if: ${{ github.event.inputs.dry_run == 'false' && steps.desc.outputs.nightly_tag != '' }} + env: + GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} + run: | + VERSION="${{ steps.checks.outputs.version }}" + NIGHTLY_TAG="${{ steps.desc.outputs.nightly_tag }}" + REPO_OWNER="${GITHUB_REPOSITORY_OWNER,,}" + IMAGE_REPO="ghcr.io/${REPO_OWNER}/${{ github.event.repository.name }}" + + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + VARIANTS=("" "-cuda" "-cuda13" "-vulkan" "-rocm" "-intel" "-musa" "-openvino") + TYPES=("full" "light" "server") + for type in "${TYPES[@]}"; do + for variant in "${VARIANTS[@]}"; do + src="${IMAGE_REPO}:${type}${variant}-${NIGHTLY_TAG}" + dst="${IMAGE_REPO}:${type}${variant}-${VERSION}" + echo "Tagging ${src} -> ${dst}" + docker buildx imagetools create --tag "${dst}" "${src}" + done + done + - name: Dry run summary if: ${{ github.event.inputs.dry_run == 'true' }} run: | diff --git a/.github/workflows/pre-tokenizer-hashes.yml b/.github/workflows/pre-tokenizer-hashes.yml index 3e440b67d9ba..bfb79f6983f2 100644 --- a/.github/workflows/pre-tokenizer-hashes.yml +++ b/.github/workflows/pre-tokenizer-hashes.yml @@ -12,7 +12,7 @@ on: jobs: pre-tokenizer-hashes: - runs-on: [self-hosted, fast] + runs-on: ubuntu-slim steps: - name: Checkout repository diff --git a/.github/workflows/python-check-requirements.yml b/.github/workflows/python-check-requirements.yml index 2c7fab40b441..e21c7da57202 100644 --- a/.github/workflows/python-check-requirements.yml +++ b/.github/workflows/python-check-requirements.yml @@ -20,7 +20,7 @@ concurrency: jobs: python-check-requirements: - runs-on: [self-hosted, CPU, fast] + runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} name: check-requirements steps: - name: Check out source repository diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 0424f372a147..1e5d64c1aee6 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -21,7 +21,7 @@ concurrency: jobs: flake8-lint: - runs-on: [self-hosted, fast] + runs-on: ubuntu-slim name: Lint steps: - name: Check out source repository diff --git a/.github/workflows/python-type-check.yml b/.github/workflows/python-type-check.yml index 14edb1a9d179..f3695be96f75 100644 --- a/.github/workflows/python-type-check.yml +++ b/.github/workflows/python-type-check.yml @@ -22,7 +22,7 @@ concurrency: jobs: python-type-check: - runs-on: [self-hosted, fast] + runs-on: ubuntu-slim name: python type-check steps: - name: Check out source repository @@ -31,7 +31,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: "3.11" - pip-install: -r requirements/requirements-all.txt ty==0.0.35 + pip-install: -r requirements/requirements-all.txt ty==0.0.78 # - name: Type-check with Pyright # uses: jakebailey/pyright-action@v2 # with: @@ -40,4 +40,4 @@ jobs: # warnings: true - name: Type-check with ty run: | - ty check --output-format=github + ty check --exit-zero-on-warning --output-format=github diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1101fef157bc..be13d9118a46 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -106,6 +106,7 @@ jobs: uses: ggml-org/ccache-action@v1.2.24 with: key: release-${{ matrix.os }}-${{ matrix.arch }} + evict-old-files: 1d - name: Build id: cmake_build @@ -190,6 +191,7 @@ jobs: uses: ggml-org/ccache-action@v1.2.24 with: key: release-${{ matrix.os }}-cpu + evict-old-files: 1d - name: Build id: cmake_build @@ -275,6 +277,7 @@ jobs: uses: ggml-org/ccache-action@v1.2.24 with: key: release-${{ matrix.os }}-vulkan + evict-old-files: 1d - name: Build id: cmake_build @@ -310,11 +313,150 @@ jobs: with: key: release-${{ matrix.os }}-vulkan + ubuntu-cuda: + name: ubuntu-cuda (${{ matrix.label }}, ${{ matrix.build }}) + needs: [check-release, ui-build] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + strategy: + matrix: + include: + # label = short version used in artifact names / release body + # cuda = full container image tag + - build: 'x64' + os: ubuntu-24.04 + cuda: '12.8.2' + label: '12.8' + defines: '-DGGML_CUDA_CUB_3DOT2=ON' + - build: 'x64' + os: ubuntu-24.04 + cuda: '13.4.1' + label: '13.4' + defines: '' + - build: 'arm64' + os: ubuntu-24.04-arm + cuda: '13.4.1' + label: '13.4' + defines: '' + + runs-on: ${{ matrix.os }} + container: nvidia/cuda:${{ matrix.cuda }}-devel-ubuntu24.04 + + permissions: + actions: write + + steps: + # the container has no git; install it before checkout so that a real git + # repository is created (the get-tag-name action and the build both need it) + - name: Install git + run: | + apt-get update + apt-get install -y --no-install-recommends git + + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + # checkout runs as the host user; in-container steps run as root, so git + # refuses to touch a repo it does not own. Mark the workspace as safe. + # use the env var: the github.workspace context holds the HOST path, + # GITHUB_WORKSPACE the container path + - name: Git safe directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Download UI build + uses: actions/download-artifact@v7 + with: + name: llama-ui.zip + path: tools/ui/dist + + - name: Dependencies + id: depends + # container jobs default to sh (dash); need bash for the [[ ]] below + shell: bash + run: | + apt-get update + apt-get install -y --no-install-recommends build-essential cmake ninja-build libssl-dev jq python3-venv + # the container ships GCC 13, which does not know the 'sme' march + # feature used by the armv9.2 CPU variant of GGML_CPU_ALL_VARIANTS + if [[ "${{ matrix.build }}" == "arm64" ]]; then + apt-get install -y --no-install-recommends gcc-14 g++-14 + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + fi + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + key: release-ubuntu-${{ matrix.os }}-cuda-${{ matrix.label }}-${{ matrix.build }} + evict-old-files: 1d + max-size: "1G" + + - name: Build + id: cmake_build + # no CMAKE_CUDA_ARCHITECTURES: use the broad default arch set from + # ggml/src/ggml-cuda/CMakeLists.txt so the release binary covers many GPUs + run: | + cmake -B build \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_CUDA=ON \ + -DGGML_CUDA_NCCL=OFF \ + ${{ env.CMAKE_ARGS }} ${{ matrix.defines }} + cmake --build build --config Release -j $(nproc) + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz + name: llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz + + # ship the CUDA runtime libraries the backend links against, mirroring + # the windows-cuda cudart zip - extract next to the binaries ($ORIGIN rpath) + - name: Pack CUDA runtime + id: pack_cuda_runtime + run: | + major="${{ matrix.label }}" + major="${major%%.*}" + mkdir -p ./cudart + # cp -L dereferences the SONAME symlinks into plain files, so the + # tarball holds exactly 3 files with no versioned duplicates + cp -L /usr/local/cuda/lib64/libcudart.so.${major} ./cudart/ + cp -L /usr/local/cuda/lib64/libcublas.so.${major} ./cudart/ + cp -L /usr/local/cuda/lib64/libcublasLt.so.${major} ./cudart/ + tar -czvf cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}," -C ./cudart . + + - name: Upload CUDA runtime + uses: actions/upload-artifact@v6 + with: + path: cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz + name: cudart-llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-ubuntu-${{ matrix.os }}-cuda-${{ matrix.label }}-${{ matrix.build }} + android-arm64: needs: [check-release, ui-build] if: ${{ needs.check-release.outputs.should_release == 'true' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # previously ubuntu-latest #permissions: # actions: write @@ -342,7 +484,7 @@ jobs: distribution: temurin - name: Setup Android SDK - uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 + uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4 with: log-accepted-android-sdk-licenses: false @@ -361,6 +503,7 @@ jobs: # uses: ggml-org/ccache-action@v1.2.24 # with: # key: release-android-arm64 + # evict-old-files: 1d - name: Build id: cmake_build @@ -401,6 +544,120 @@ jobs: path: llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz name: llama-bin-android-arm64.tar.gz + android-arm64-snapdragon: + needs: [check-release, ui-build] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: ubuntu-latest + container: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7' + + defaults: + run: + shell: bash + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + # checkout runs as the host user; in-container steps run as root, so git + # refuses to touch a repo it does not own. Mark the workspace as safe. + - name: Git safe directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Download UI build + uses: actions/download-artifact@v7 + with: + name: llama-ui.zip + path: tools/ui/dist + + - name: Build + id: cmake_build + run: | + cp docs/backend/snapdragon/CMakeUserPresets.json . + cmake --preset arm64-android-snapdragon-release -B build \ + -DCMAKE_INSTALL_RPATH='$ORIGIN;$ORIGIN/../lib' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DLLAMA_BUILD_BORINGSSL=ON \ + ${{ env.CMAKE_ARGS }} + cmake --build build -j $(nproc) + cmake --install build --prefix pkg-snapdragon/llama.cpp + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + run: | + cp LICENSE pkg-snapdragon/llama.cpp/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-android-arm64-snapdragon.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C pkg-snapdragon/llama.cpp . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-android-arm64-snapdragon.tar.gz + name: llama-bin-android-arm64-snapdragon.tar.gz + + linux-arm64-snapdragon: + needs: [check-release, ui-build] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: ubuntu-latest + container: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7' + + defaults: + run: + shell: bash + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + # checkout runs as the host user; in-container steps run as root, so git + # refuses to touch a repo it does not own. Mark the workspace as safe. + - name: Git safe directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Download UI build + uses: actions/download-artifact@v7 + with: + name: llama-ui.zip + path: tools/ui/dist + + - name: Build + id: cmake_build + run: | + cp docs/backend/snapdragon/CMakeUserPresets.json . + cmake --preset arm64-linux-snapdragon-release -B build -DGGML_OPENCL=ON \ + -DCMAKE_INSTALL_RPATH='$ORIGIN;$ORIGIN/../lib' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DLLAMA_BUILD_BORINGSSL=ON \ + ${{ env.CMAKE_ARGS }} + cmake --build build -j $(nproc) + cmake --install build --prefix pkg-snapdragon/llama.cpp + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + run: | + cp LICENSE pkg-snapdragon/llama.cpp/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C pkg-snapdragon/llama.cpp . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz + name: llama-bin-linux-arm64-snapdragon.tar.gz + ubuntu-24-openvino: needs: [check-release, ui-build] if: ${{ needs.check-release.outputs.should_release == 'true' }} @@ -415,8 +672,8 @@ jobs: env: # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.3.1" - OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d" + OPENVINO_VERSION_MAJOR: "2026.4" + OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3" steps: - name: Set OpenVINO version output @@ -439,6 +696,7 @@ jobs: uses: ggml-org/ccache-action@v1.2.24 with: key: release-ubuntu-24.04-openvino-release-no-preset-v1 + evict-old-files: 1d - name: Dependencies run: | @@ -529,8 +787,8 @@ jobs: env: # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.3.1" - OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d" + OPENVINO_VERSION_MAJOR: "2026.4" + OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3" steps: - name: Set OpenVINO version output @@ -682,6 +940,7 @@ jobs: uses: ggml-org/ccache-action@v1.2.24 with: key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu + evict-old-files: 1d - name: Build shell: cmd @@ -926,6 +1185,7 @@ jobs: # uses: ggml-org/ccache-action@v1.2.24 # with: # key: release-windows-2025-${{ matrix.arch }}-${{ matrix.backend }} + # evict-old-files: 1d - name: Install OpenCL Headers and Libs id: install_opencl @@ -987,7 +1247,7 @@ jobs: - cuda: '12.4' arch: x64 defines: '-DGGML_CUDA_CUB_3DOT2=ON' - - cuda: '13.3' + - cuda: '13.4' arch: x64 defines: '' - cuda: '13.4' @@ -1014,6 +1274,7 @@ jobs: uses: ggml-org/ccache-action@v1.2.24 with: key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }} + evict-old-files: 1d - name: Build id: cmake_build @@ -1110,6 +1371,7 @@ jobs: uses: ggml-org/ccache-action@v1.2.24 with: key: release-windows-2022-x64-sycl + evict-old-files: 1d - name: Build id: cmake_build @@ -1228,6 +1490,7 @@ jobs: uses: ggml-org/ccache-action@v1.2.24 with: key: release-ubuntu-24.04-sycl-${{ matrix.build }} + evict-old-files: 1d - name: Build id: cmake_build @@ -1280,7 +1543,7 @@ jobs: matrix: include: - ROCM_VERSION: "10.0.0" - gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201" + gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201" build: 'x64' steps: @@ -1572,9 +1835,12 @@ jobs: - ubuntu-24-rocm - ubuntu-cpu - ubuntu-vulkan + - ubuntu-cuda - ubuntu-24-openvino - ubuntu-24-sycl - android-arm64 + - android-arm64-snapdragon + - linux-arm64-snapdragon - macos-cpu - ios-xcode #- openEuler-cann @@ -1703,21 +1969,26 @@ jobs: - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz) - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz) - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz) + - [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) + - [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) + - [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) - [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz) - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz) - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz) - [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz) + - [Linux arm64 (Snapdragon: CPU, Adreno GPU, Hexagon NPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz) - [setup guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/linux.md) **Android:** - [Android arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz) + - [Android arm64 (Snapdragon: CPU, Adreno GPU, Hexagon NPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64-snapdragon.tar.gz) - [setup guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/README.md) **Windows:** - [Windows x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-x64.zip) - [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip) - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip) - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip) - - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.3-x64.zip) - [CUDA 13.3 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.3-x64.zip) - - [Windows arm64 (CUDA 13) (preview)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip) + - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip) + - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip) - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip) - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip) - [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip) diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.yml index 77549ee8717a..8ff285a42afe 100644 --- a/.github/workflows/server-sanitize.yml +++ b/.github/workflows/server-sanitize.yml @@ -32,6 +32,8 @@ on: ] env: + # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} LLAMA_ARG_LOG_COLORS: 1 LLAMA_ARG_LOG_PREFIX: 1 LLAMA_ARG_LOG_TIMESTAMPS: 1 @@ -43,7 +45,7 @@ concurrency: jobs: server: - runs-on: [self-hosted, CPU, Linux, llama-server] + runs-on: hf-jobs-cpu-xl strategy: matrix: @@ -52,20 +54,6 @@ jobs: fail-fast: false steps: - #- name: Dependencies - # id: depends - # run: | - # sudo apt-get update - # sudo apt-get -y install \ - # build-essential \ - # xxd \ - # git \ - # cmake \ - # curl \ - # wget \ - # language-pack-en \ - # libssl-dev - - name: Clone id: checkout uses: actions/checkout@v6 @@ -73,6 +61,24 @@ jobs: fetch-depth: 0 ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y build-essential cmake python3-full + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + restore: false + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + with: + key: server-sanitize-${{ matrix.sanitizer }} + folder: llama.cpp + hf_bucket: ggml-org/cache + - name: Build id: cmake_build run: | @@ -87,9 +93,17 @@ jobs: -DLLAMA_SANITIZE_UNDEFINED=${{ matrix.sanitizer == 'UNDEFINED' }} cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target llama-server - - name: Python setup - id: setup_python - uses: actions/setup-python@v7 + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: server-sanitize-${{ matrix.sanitizer }} + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ggml-org/cache + save: true - name: Install Python dependencies run: | @@ -102,7 +116,6 @@ jobs: run: | source .venv/bin/activate cd tools/server/tests - export ${{ matrix.extra_args }} PYTEST_WORKERS=1 ./tests.sh - name: Slow tests @@ -111,5 +124,4 @@ jobs: run: | source .venv/bin/activate cd tools/server/tests - export ${{ matrix.extra_args }} PYTEST_WORKERS=1 SLOW_TESTS=1 ./tests.sh diff --git a/.github/workflows/server-self-hosted.yml b/.github/workflows/server-self-hosted.yml index d9ad2fcd0dc0..8dc4637c4570 100644 --- a/.github/workflows/server-self-hosted.yml +++ b/.github/workflows/server-self-hosted.yml @@ -72,7 +72,7 @@ jobs: run: | cd tools/server/tests source venv/bin/activate - ./tests.sh + PYTEST_WORKERS=1 ./tests.sh - name: Tests (GPUx1, backend-sampling) id: server_integration_tests_backend_sampling @@ -81,7 +81,7 @@ jobs: cd tools/server/tests source venv/bin/activate export LLAMA_ARG_BACKEND_SAMPLING=1 - ./tests.sh + PYTEST_WORKERS=1 ./tests.sh - name: Tests (GPUx2) id: server_integration_tests_gpu2 @@ -90,7 +90,7 @@ jobs: cd tools/server/tests source venv/bin/activate export GGML_METAL_DEVICES=2 - ./tests.sh + PYTEST_WORKERS=1 ./tests.sh - name: Tests (GPUx2, backend-sampling) id: server_integration_tests_gpu2_backend_sampling @@ -99,7 +99,7 @@ jobs: cd tools/server/tests source venv/bin/activate export GGML_METAL_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1 - ./tests.sh + PYTEST_WORKERS=1 ./tests.sh server-cuda: runs-on: "hf-jobs-t4-small:cuda13" @@ -162,7 +162,7 @@ jobs: run: | cd tools/server/tests source venv/bin/activate - ./tests.sh + PYTEST_WORKERS=1 ./tests.sh - name: Tests (GPUx1, backend-sampling) id: server_integration_tests_backend_sampling @@ -171,7 +171,7 @@ jobs: cd tools/server/tests source venv/bin/activate export LLAMA_ARG_BACKEND_SAMPLING=1 - ./tests.sh + PYTEST_WORKERS=1 ./tests.sh - name: Tests (GPUx2) id: server_integration_tests_gpu2 @@ -180,7 +180,7 @@ jobs: cd tools/server/tests source venv/bin/activate export GGML_CUDA_DEVICES=2 - ./tests.sh + PYTEST_WORKERS=1 ./tests.sh - name: Tests (GPUx2, backend-sampling) id: server_integration_tests_gpu2_backend_sampling @@ -189,10 +189,10 @@ jobs: cd tools/server/tests source venv/bin/activate export GGML_CUDA_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1 - ./tests.sh + PYTEST_WORKERS=1 ./tests.sh server-kleidiai: - runs-on: ah-ubuntu_22_04-c8g_8x + runs-on: ah-ubuntu_24_04-c8g_8x steps: - name: Clone @@ -232,7 +232,7 @@ jobs: - name: Build id: cmake_build run: | - cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON + cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON -DLLAMA_FATAL_WARNINGS=ON cmake --build build --config Release -j $(nproc) --target llama-server - name: Python setup diff --git a/.github/workflows/ui-build-self-hosted.yml b/.github/workflows/ui-build-self-hosted.yml index 390a2f35f79d..e93a89003b23 100644 --- a/.github/workflows/ui-build-self-hosted.yml +++ b/.github/workflows/ui-build-self-hosted.yml @@ -17,8 +17,9 @@ jobs: uses: actions/setup-node@v6 with: node-version: "24" - cache: "npm" - cache-dependency-path: "tools/ui/package-lock.json" + # cache: "npm" + # cache-dependency-path: "tools/ui/package-lock.json" + package-manager-cache: false - name: Install dependencies run: npm ci diff --git a/.github/workflows/ui-build.yml b/.github/workflows/ui-build.yml index 3fbd90c11cf6..cbadaa9e76d1 100644 --- a/.github/workflows/ui-build.yml +++ b/.github/workflows/ui-build.yml @@ -33,8 +33,9 @@ jobs: uses: actions/setup-node@v6 with: node-version: "24" - cache: "npm" - cache-dependency-path: "tools/ui/package-lock.json" + # cache: "npm" + # cache-dependency-path: "tools/ui/package-lock.json" + package-manager-cache: false - name: Install dependencies run: npm ci diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 00a0804af389..f395c0b52873 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -57,8 +57,9 @@ jobs: uses: actions/setup-node@v6 with: node-version: "24" - cache: "npm" - cache-dependency-path: "tools/ui/package-lock.json" + # cache: "npm" + # cache-dependency-path: "tools/ui/package-lock.json" + package-manager-cache: false - name: Download built UI artifacts uses: actions/download-artifact@v6 @@ -114,8 +115,9 @@ jobs: uses: actions/setup-node@v6 with: node-version: "24" - cache: "npm" - cache-dependency-path: "tools/ui/package-lock.json" + # cache: "npm" + # cache-dependency-path: "tools/ui/package-lock.json" + package-manager-cache: false - name: Install dependencies id: setup diff --git a/.github/workflows/update-ops-docs.yml b/.github/workflows/update-ops-docs.yml index 6e8bc1aa07c2..bb01c7e56bab 100644 --- a/.github/workflows/update-ops-docs.yml +++ b/.github/workflows/update-ops-docs.yml @@ -16,7 +16,7 @@ on: jobs: update-ops-docs: - runs-on: [self-hosted, fast, ARM64] + runs-on: ubuntu-slim steps: - name: Checkout repository diff --git a/.github/workflows/winget.yml b/.github/workflows/winget.yml index c0a814f3adbf..7af2e9b10b3a 100644 --- a/.github/workflows/winget.yml +++ b/.github/workflows/winget.yml @@ -8,7 +8,7 @@ on: jobs: update: name: Update Winget Package - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # previously ubuntu-latest if: github.repository_owner == 'ggml-org' steps: diff --git a/.pi/gg/SYSTEM.md b/.pi/gg/SYSTEM.md index 47883081cfad..bd308ea963bc 100644 --- a/.pi/gg/SYSTEM.md +++ b/.pi/gg/SYSTEM.md @@ -6,6 +6,7 @@ General: - PR and commit titles format: ` : `. Lookup recents for examples - Don't try to build or run the code unless you are explicitly asked to do so - Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources +- When [MODEL] is needed, first try to get it from the `PI_MODEL_NAME` env var before asking the user Coding: - When in doubt, always refer to the CONTRIBUTING.md file of the project @@ -20,8 +21,9 @@ Pull requests (PRs): - Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line) - When creating a pull request, look for the repository's PR template and follow it - For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]" -- Ask the user to tell you what model was used and write it in place of [MODEL] +- If `PI_MODEL_NAME` env var is not set, ask the user to tell you what model was used and write it in place of [MODEL] - Always create the pull requests in draft mode +- Never reply to review comments or post comments on issues/PRs without explicit permission from the user Commits: - On every commit that you make, include a "Assisted-by: pi:llama.cpp/[MODEL]" tag diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e75d2e2afbf..a80f776073ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ include(CheckIncludeFileCXX) ### llama.cpp version set(LLAMA_VERSION_MAJOR 0) -set(LLAMA_VERSION_MINOR 4) +set(LLAMA_VERSION_MINOR 5) set(LLAMA_VERSION_PATCH 0) set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}") @@ -197,6 +197,16 @@ llama_option_depr(WARNING LLAMA_CURL) include("cmake/license.cmake") license_add_file("llama.cpp" "LICENSE") +# +# compile options +# + +# clang stores the modification time of the precompiled header sources inside the +# header and rejects it when they differ, so the timestamp is left out of it +add_compile_options( + "$<$<COMPILE_LANG_AND_ID:C,Clang,IntelLLVM>:SHELL:-Xclang -fno-pch-timestamp>" + "$<$<COMPILE_LANG_AND_ID:CXX,Clang,IntelLLVM>:SHELL:-Xclang -fno-pch-timestamp>") + # # 3rd-party # diff --git a/CODEOWNERS b/CODEOWNERS index 725a1b7e6534..0fbfcfdd1236 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -97,7 +97,6 @@ /src/models/ @CISC /tests/ @ggerganov /tests/test-chat.* @pwilkin -/tests/test-llama-archs.cpp @JohannesGaessler /tools/batched-bench/ @ggerganov /tools/cli/ @ngxson /tools/completion/ @ggerganov diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6aac3cb878da..59ec3f311b5d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,8 +20,8 @@ If AI is used to generate any portion of the code, contributors must adhere to t 1. Explicitly disclose the manner in which AI was employed. 2. Check for an existing PR addressing the same change; if one exists, comment there to work with its author instead of opening a duplicate. -3. Perform a comprehensive manual review prior to submitting the pull request. -4. Be prepared to explain every line of code they submitted when asked about it by a maintainer. +3. Perform a comprehensive manual review prior to submitting the pull request. A proper code review usually takes something like one hour per 200-400 LOC and you should be spending **at least that much time on code review alone**. +4. Be prepared to explain every line of code you submit when asked about it by a maintainer. 5. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...). For more info, please refer to the [AGENTS.md](AGENTS.md) file. diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index 3450ff49000f..0b044228aff8 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -16,7 +16,7 @@ target_link_libraries(${TARGET} PRIVATE target_compile_features(${TARGET} PRIVATE cxx_std_17) # Automatically add all files from the 'licenses' directory -file(GLOB EXTRA_LICENSES "${CMAKE_SOURCE_DIR}/licenses/LICENSE-*") +file(GLOB EXTRA_LICENSES "${PROJECT_SOURCE_DIR}/licenses/LICENSE-*") foreach(FILE_PATH ${EXTRA_LICENSES}) get_filename_component(FILE_NAME "${FILE_PATH}" NAME) diff --git a/ci/run.sh b/ci/run.sh index 5463597274ec..0595fac5a1b8 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -58,8 +58,6 @@ if [ -n "${GG_BUILD_ROCM}" ] && [ -n "${GITHUB_RUN_ID}" ]; then fi rm -f $OUT/*.log -rm -f $OUT/*.exit -rm -f $OUT/*.md sd=`dirname $0` cd $sd/../ @@ -190,7 +188,7 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON" # TODO: fix failing tests on OpenVINO backend - CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state" + CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-save-load-state" fi ## helpers @@ -211,10 +209,6 @@ function gg_wget { cd $cwd } -function gg_printf { - printf -- "$@" >> $OUT/README.md -} - function gg_run { ci=$1 @@ -223,13 +217,10 @@ function gg_run { gg_run_$ci | tee $OUT/$ci.log cur=$? - echo "$cur" > $OUT/$ci.exit set +x set +o pipefail - gg_sum_$ci - ret=$((ret | cur)) } @@ -250,22 +241,11 @@ function gg_run_ctest_debug { (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log (time cmake --build . --config Debug -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log - (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-backend-ops|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log + (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log set +e } -function gg_sum_ctest_debug { - gg_printf '### %s\n\n' "${ci}" - - gg_printf 'Runs ctest in debug mode\n' - gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" - gg_printf '```\n' - gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)" - gg_printf '```\n' - gg_printf '\n' -} - # ctest_release function gg_run_ctest_release { @@ -290,16 +270,6 @@ function gg_run_ctest_release { set +e } -function gg_sum_ctest_release { - gg_printf '### %s\n\n' "${ci}" - - gg_printf 'Runs ctest in release mode\n' - gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" - gg_printf '```\n' - gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)" - gg_printf '```\n' -} - # test_llama_archs_tensor_split function gg_run_test_llama_archs_tensor_split { @@ -324,14 +294,23 @@ function gg_run_test_llama_archs_tensor_split { set +e } -function gg_sum_test_llama_archs_tensor_split { - gg_printf '### %s\n\n' "${ci}" +# test_llama_archs_models + +function gg_run_test_llama_archs_models { + cd ${SRC} + + set -e + + # TODO: fix and re-enable `test-llama-archs` on OpenVINO + # TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists + if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then + rm -rf build-ci-models && mkdir -p build-ci-models + + # generate the dummy models used by the model-dependent tests + ./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1 + fi - gg_printf 'Runs test-llama-archs with 1 to 4 devices\n' - gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" - gg_printf '```\n' - gg_printf '%s\n' "$(cat $OUT/${ci}.log)" - gg_printf '```\n' + set +e } # test_scripts @@ -347,17 +326,6 @@ function gg_run_test_scripts { set +e } -function gg_sum_test_scripts { - gg_printf '### %s\n\n' "${ci}" - - gg_printf 'Runs test scripts\n' - gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" - gg_printf '```\n' - gg_printf '%s\n' "$(cat $OUT/${ci}-scripts.log)" - gg_printf '```\n' - gg_printf '\n' -} - function gg_get_model { #local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-f16.gguf" local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-q4_0.gguf" @@ -401,26 +369,6 @@ function gg_run_ctest_with_model_release { cd .. } -function gg_sum_ctest_with_model_debug { - gg_printf '### %s\n\n' "${ci}" - - gg_printf 'Runs ctest with model files in debug mode\n' - gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" - gg_printf '```\n' - gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)" - gg_printf '```\n' -} - -function gg_sum_ctest_with_model_release { - gg_printf '### %s\n\n' "${ci}" - - gg_printf 'Runs ctest with model files in release mode\n' - gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" - gg_printf '```\n' - gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)" - gg_printf '```\n' -} - # qwen3_0_6b function gg_run_qwen3_0_6b { @@ -525,50 +473,24 @@ function gg_run_qwen3_0_6b { return 0 } - check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")" if [ -z ${GG_BUILD_NO_BF16} ]; then - check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")" fi - check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - #check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log # note: ppl > 20.0 for this quant and model - check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - - cat $OUT/${ci}-imatrix.log | grep "Final" >> $OUT/${ci}-imatrix-sum.log + check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" + check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" + check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" + check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" + check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" + #check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" # note: ppl > 20.0 for this quant and model + check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" + check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" + check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" + check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" set +e } -function gg_sum_qwen3_0_6b { - gg_printf '### %s\n\n' "${ci}" - - gg_printf 'Qwen3 0.6B:\n' - gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" - gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)" - gg_printf '- imatrix:\n```\n%s\n```\n' "$(cat $OUT/${ci}-imatrix-sum.log)" - gg_printf '- f16:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)" - if [ -z ${GG_BUILD_NO_BF16} ]; then - gg_printf '- bf16:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-bf16.log)" - fi - gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)" - gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)" - gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)" - gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)" - gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)" - gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)" - gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)" - gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)" - gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)" - gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)" - gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)" -} - # bge-small function gg_run_embd_bge_small { @@ -610,15 +532,6 @@ function gg_run_embd_bge_small { set +e } -function gg_sum_embd_bge_small { - gg_printf '### %s\n\n' "${ci}" - - gg_printf 'BGE Small (BERT):\n' - gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" - gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)" - gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)" -} - # rerank_tiny function gg_run_rerank_tiny { @@ -675,91 +588,100 @@ function gg_run_rerank_tiny { set +e } -function gg_sum_rerank_tiny { - gg_printf '### %s\n\n' "${ci}" - - gg_printf 'Rerank Tiny (Jina):\n' - gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" - gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-rk-f16.log)" -} - function gg_check_build_requirements { if ! command -v git &> /dev/null; then - gg_printf 'git not found, please install\n' + echo 'git not found, please install' exit 1 fi if ! command -v git-lfs &> /dev/null; then - gg_printf 'git-lfs not found, please install\n' + echo 'git-lfs not found, please install' exit 1 fi if ! git config --get filter.lfs.clean &> /dev/null; then - gg_printf 'git-lfs not initialized, please run `git lfs install`\n' + echo 'git-lfs not initialized, please run `git lfs install`' exit 1 fi if ! command -v wget &> /dev/null; then - gg_printf 'wget not found, please install\n' + echo 'wget not found, please install' exit 1 fi if ! command -v python3 &> /dev/null; then - gg_printf 'python3 not found, please install\n' + echo 'python3 not found, please install' exit 1 fi if ! command -v pip3 &> /dev/null; then - gg_printf 'pip3 not found, please install\n' + echo 'pip3 not found, please install' exit 1 fi if ! python3 -m ensurepip --help &> /dev/null; then - gg_printf 'ensurepip not found, please install python3-venv package\n' + echo 'ensurepip not found, please install python3-venv package' exit 1 fi if ! command -v cmake &> /dev/null; then - gg_printf 'cmake not found, please install\n' + echo 'cmake not found, please install' exit 1 fi if ! command -v ccache &> /dev/null; then - gg_printf 'ccache not found, please consider installing for faster builds\n' + echo 'ccache not found, please consider installing for faster builds' fi if ! command -v ctest &> /dev/null; then - gg_printf 'ctest not found, please install\n' + echo 'ctest not found, please install' exit 1 fi if ! command -v unzip &> /dev/null; then - gg_printf 'unzip not found, please install\n' + echo 'unzip not found, please install' exit 1 fi } -function gg_run_test_backend_ops_cpu { +function gg_run_test_backend_ops { cd ${SRC} cd build-ci-release set -e - (time ./bin/test-backend-ops -b CPU ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops-cpu.log + local n_jobs=$(nproc) + if [ "${n_jobs}" -gt 2 ]; then + n_jobs=2 + fi + local args_extra="-j ${n_jobs}" - set +e -} + # TODO: fix multi-threaded for ROCm + # https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865 + if [ ! -z ${GG_BUILD_ROCM} ]; then + args_extra="" + fi -function gg_sum_test_backend_ops_cpu { - gg_printf '### %s\n\n' "${ci}" + # TODO: MoltenVK bug? + # https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897 + if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then + args_extra="" + fi - gg_printf 'Runs test-backend-ops for CPU backend\n' - gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" - gg_printf '```\n' - gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops-cpu.log)" - gg_printf '```\n' - gg_printf '\n' + # TODO: OpenVINO GPU plugin crashes (CL_OUT_OF_RESOURCES) with 2 concurrent workers on GPU. + if [ ! -z "${GG_BUILD_OPENVINO}" ] && [ "${GGML_OPENVINO_DEVICE:-}" = "GPU" ]; then + args_extra="" + fi + + # TODO: reduce the test-backend-ops timeout to 1800s + if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then + (time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log + else + (time timeout 3600 ./bin/test-backend-ops ${args_extra} ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log + fi + + set +e } ## main @@ -790,11 +712,10 @@ ret=0 test $ret -eq 0 && gg_run ctest_debug test $ret -eq 0 && gg_run ctest_release -test $ret -eq 0 && gg_run test_llama_archs_tensor_split +test $ret -eq 0 && gg_run test_backend_ops -if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then - test $ret -eq 0 && gg_run test_backend_ops_cpu -fi +test $ret -eq 0 && gg_run test_llama_archs_models +test $ret -eq 0 && gg_run test_llama_archs_tensor_split if [ -z ${GG_BUILD_LOW_PERF} ]; then test $ret -eq 0 && gg_run embd_bge_small @@ -810,6 +731,4 @@ if [ -z ${GG_BUILD_LOW_PERF} ]; then test $ret -eq 0 && gg_run ctest_with_model_release fi -cat $OUT/README.md - exit $ret diff --git a/cmake/llama-config.cmake.in b/cmake/llama-config.cmake.in index 6db73577ae6d..6500203b145f 100644 --- a/cmake/llama-config.cmake.in +++ b/cmake/llama-config.cmake.in @@ -17,14 +17,16 @@ find_library(llama_LIBRARY llama NO_CMAKE_FIND_ROOT_PATH ) -add_library(llama UNKNOWN IMPORTED) -set_target_properties(llama - PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${LLAMA_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "ggml::ggml;ggml::ggml-base;" - IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" - IMPORTED_LOCATION "${llama_LIBRARY}" - INTERFACE_COMPILE_FEATURES c_std_90 - POSITION_INDEPENDENT_CODE ON) +if(NOT TARGET llama) + add_library(llama UNKNOWN IMPORTED) + set_target_properties(llama + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${LLAMA_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "ggml::ggml;ggml::ggml-base;" + IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" + IMPORTED_LOCATION "${llama_LIBRARY}" + INTERFACE_COMPILE_FEATURES c_std_90 + POSITION_INDEPENDENT_CODE ON) +endif() check_required_components(Llama) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index f7d429c27512..e387443131d4 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -53,7 +53,10 @@ endif() set(TARGET llama-common) +include(parsers/sources.cmake) + add_library(${TARGET} + ${LLAMA_CHAT_PARSERS_SOURCES} arg.cpp arg.h base64.hpp @@ -81,6 +84,8 @@ add_library(${TARGET} imatrix-loader.cpp imatrix-loader.h json-schema-to-grammar.cpp + json-schema.cpp + json-schema.h json.cpp json.h llguidance.cpp diff --git a/common/arg.cpp b/common/arg.cpp index 408d8b88b62b..2bcfcd580e92 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -881,17 +881,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context arg.c_str(), e.what(), opt.to_string().c_str())); } } - - // TODO: remove this check after deprecating --mmap|mlock|dio - auto has_arg = [&](std::initializer_list<const char *> names) { - return std::any_of(names.begin(), names.end(), [&](const char * name) { - return seen_args.count(name); - }); - }; - if (has_arg({"-lm", "--load-mode"}) && - has_arg({"--mlock", "--mmap", "--no-mmap", "-dio", "--direct-io", "-ndio", "--no-direct-io"})) { - LOG_WRN("DEPRECATED: `--load-mode` and `--mlock`/`--mmap`/`--direct-io` should not be combined; only the last flag on the command line will take effect\n"); - } }; // parse all CLI args now, so that -hf is available below for remote preset resolution @@ -925,6 +914,12 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context "--no-kv-tier-paged-blocks to opt out)\n", __func__); } + // default the mmproj device to the global device selection if not set explicitly with -mmdev + if (params.mmproj_use_gpu && params.mmproj_device == nullptr && !params.devices.empty()) { + params.mmproj_device = params.devices.front(); + params.mmproj_use_gpu = params.mmproj_device != nullptr; + } + if (params.prompt_cache_all && (params.interactive || params.interactive_first)) { throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n"); } @@ -2339,7 +2334,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sampling.temp = std::max(params.sampling.temp, 0.0f); params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TEMP; } - ).set_sampling()); + ).set_sampling().set_env("LLAMA_ARG_TEMPERATURE")); add_opt(common_arg( {"--top-k"}, "N", string_format("top-k sampling (default: %d, 0 = disabled)", params.sampling.top_k), @@ -2355,7 +2350,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sampling.top_p = std::stof(value); params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TOP_P; } - ).set_sampling()); + ).set_sampling().set_env("LLAMA_ARG_TOP_P")); add_opt(common_arg( {"--min-p"}, "N", string_format("min-p sampling (default: %.2f, 0.0 = disabled)", (double)params.sampling.min_p), @@ -2363,7 +2358,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sampling.min_p = std::stof(value); params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIN_P; } - ).set_sampling()); + ).set_sampling().set_env("LLAMA_ARG_MIN_P")); add_opt(common_arg( {"--top-nsigma", "--top-n-sigma"}, "N", string_format("top-n-sigma sampling (default: %.2f, -1.0 = disabled)", params.sampling.top_n_sigma), @@ -2419,7 +2414,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sampling.penalty_repeat = penalty_repeat; params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT; } - ).set_sampling()); + ).set_sampling().set_env("LLAMA_ARG_REPEAT_PENALTY")); add_opt(common_arg( {"--presence-penalty"}, "N", string_format("repeat alpha presence penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_present), @@ -2430,7 +2425,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.sampling.penalty_present = penalty_present; } - ).set_sampling()); + ).set_sampling().set_env("LLAMA_ARG_PRESENCE_PENALTY")); add_opt(common_arg( {"--frequency-penalty"}, "N", string_format("repeat alpha frequency penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_freq), @@ -2441,7 +2436,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.sampling.penalty_freq = penalty_freq; } - ).set_sampling()); + ).set_sampling().set_env("LLAMA_ARG_FREQUENCY_PENALTY")); add_opt(common_arg( {"--dry-multiplier"}, "N", string_format("set DRY sampling multiplier (default: %.2f, 0.0 = disabled)", (double)params.sampling.dry_multiplier), @@ -2600,14 +2595,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_sampling()); add_opt(common_arg( {"-j", "--json-schema"}, "SCHEMA", - "JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead", + "JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object", [](common_params & params, const std::string & value) { params.sampling.grammar = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, json_schema_to_grammar(json::parse(value))}; } ).set_sampling()); add_opt(common_arg( {"-jf", "--json-schema-file"}, "FILE", - "File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead", + "File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object", [](common_params & params, const std::string & value) { std::ifstream file(value); if (!file) { @@ -2961,7 +2956,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex add_opt(common_arg( // note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet {"-mmdev", "--mmproj-device"}, "DEVICE", - "device to use for multimodal projector (none = don't offload, default: auto)\n" + "device to use for multimodal projector (none = don't offload, default: follows --device)\n" "use --list-devices to see a list of available devices", [](common_params & params, const std::string & value) { if (value == "none") { @@ -3039,32 +3034,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } ).set_env("LLAMA_ARG_RPC")); } - add_opt(common_arg( - {"--mlock"}, - "DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing", - [](common_params & params) { - LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n"); - params.load_mode = LLAMA_LOAD_MODE_MLOCK; - } - ).set_env("LLAMA_ARG_MLOCK")); - add_opt(common_arg( - {"--mmap"}, - {"--no-mmap"}, - "DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)", - [](common_params & params, bool value) { - LOG_WRN("DEPRECATED: --mmap and --no-mmap are deprecated. use --load-mode mmap instead\n"); - params.load_mode = value ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE; - } - ).set_env("LLAMA_ARG_MMAP")); - add_opt(common_arg( - {"-dio", "--direct-io"}, - {"-ndio", "--no-direct-io"}, - "DEPRECATED in favor of `--load-mode`: use DirectIO if available", - [](common_params & params, bool value) { - LOG_WRN("DEPRECATED: --direct-io and --no-direct-io are deprecated. use --load-mode dio instead\n"); - params.load_mode = value ? LLAMA_LOAD_MODE_DIRECT_IO : LLAMA_LOAD_MODE_NONE; - } - ).set_env("LLAMA_ARG_DIO")); add_opt(common_arg( {"-lm", "--load-mode"}, "MODE", "model loading mode (default: auto)\n" @@ -3930,9 +3899,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_examples({LLAMA_EXAMPLE_EMBEDDING})); add_opt(common_arg( {"--host"}, "HOST", - string_format("ip address to listen, or bind to an UNIX socket if the address ends with .sock (default: %s)", params.hostname.c_str()), + string_format("IP addresses to listen on, comma-separated, or UNIX socket paths ending in .sock; with multiple TCP addresses, :: binds IPv6 only; overlapping addresses result in undefined behavior (default: %s)", params.hostnames[0].c_str()), [](common_params & params, const std::string & value) { - params.hostname = value; + params.hostnames.clear(); + for (auto & host : parse_csv_row(value)) { + host = string_strip(host); + if (!host.empty()) { + params.hostnames.push_back(host); + } + } + if (params.hostnames.empty()) { + throw std::invalid_argument("--host requires at least one address"); + } } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HOST")); add_opt(common_arg( @@ -4511,6 +4489,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex common_log_set_file(common_log_main(), value.c_str()); } ).set_env("LLAMA_ARG_LOG_FILE")); + add_opt(common_arg( + {"--log-jsonl"}, + {"--no-log-jsonl"}, + "Log as JSONL (one JSON object per line) to stdout, this also disables colored logging (default: disabled)", + [](common_params &, bool value) { + common_log_set_jsonl(value); + } + ).set_env("LLAMA_ARG_LOG_JSONL")); add_opt(common_arg( {"--log-prompts-dir"}, "PATH", "Log prompts to directory (auto-created if not present; only used for debugging, default: disabled)", @@ -4863,7 +4849,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING")); add_opt(common_arg( {"--spec-draft-device", "-devd", "--device-draft"}, "<dev1,dev2,..>", - "comma-separated list of devices to use for offloading the draft model (none = don't offload)\n" + "comma-separated list of devices to use for offloading the draft model (none = don't offload, default: follows --device)\n" "use --list-devices to see a list of available devices", [](common_params & params, const std::string & value) { params.speculative.draft.devices = parse_device_list(value); diff --git a/common/arg.h b/common/arg.h index 421bc295fc21..203d1b4e11b8 100644 --- a/common/arg.h +++ b/common/arg.h @@ -122,6 +122,8 @@ struct common_params_context { // parse input arguments from CLI // if one argument has invalid value, it will automatically display usage of the specific argument (and not the full usage message) +// TODO: this function can load ggml backend (by calling llama_support_rpc) +// this is a side-effect that should be avoided bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr); // load all backends and print the list of available (non-CPU) devices to stdout diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index d7e117e4d98b..b78789d8c0df 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -5,6 +5,7 @@ #include "common.h" #include "json-schema-to-grammar.h" #include "log.h" +#include "parsers/parsers.h" #include "peg-parser.h" #include <stdexcept> @@ -12,16 +13,6 @@ using json = common_json; -// Helper to iterate over tools/functions -static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) { - for (const auto & tool : tools) { - if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) { - continue; - } - fn(tool); - } -} - namespace autoparser { parser_build_context::parser_build_context(common_chat_peg_builder & p, const generation_params & inputs) : @@ -87,15 +78,6 @@ common_chat_params peg_generator::generate_parser(const common_chat_template & if (include_grammar) { data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } parser.build_grammar(builder, data.grammar_lazy); }); @@ -312,7 +294,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context foreach_function(inputs.tools, [&](const json & tool) { const auto & func = tool.at("function"); std::string name = func.at("name"); - const auto & schema = func.contains("parameters") ? func.at("parameters") : json::object(); + const auto schema = common_chat_tool_parameters(func); // Build call_id parser based on position (if supported) bool have_call_id = false; @@ -383,43 +365,31 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte common_peg_parser tool_choice = p.choice(); foreach_function(inputs.tools, [&](const json & tool) { - const auto & func = tool.at("function"); - std::string name = func.at("name"); - auto params = func.contains("parameters") ? func.at("parameters") : json::object(); - const auto & properties = params.contains("properties") ? params.at("properties") : json::object(); - - std::set<std::string> required; - if (params.contains("required")) { - required = params.at("required").get<std::set<std::string>>(); - } - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); + const auto & func = tool.at("function"); + std::string name = func.at("name"); // Build parser for each argument, separating required and optional std::vector<common_peg_parser> required_parsers; std::vector<common_peg_parser> optional_parsers; - for (const auto & [param_name, param_schema] : properties.items()) { - bool is_required = required.find(param_name) != required.end(); - + foreach_parameter(func, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) { auto arg = - p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) + + p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) + arguments.name_suffix) + arguments.value_prefix + - (schema_info.resolves_to_string(param_schema) ? + (param.schema->may_be_string() ? p.ac(p.tool_arg_string_value(until_suffix) + p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) : (p.tool_arg_json_value(p.schema( - p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) + + p.json(), "tool-" + name + "-arg-" + param.name + "-schema", doc, *param.schema)) + p.tool_arg_close(p.literal(arguments.value_suffix))))); - auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); - if (is_required) { + auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg); + if (param.required) { required_parsers.push_back(named_arg); } else { optional_parsers.push_back(named_arg); } - } + }); // Build required arg sequence in definition order common_peg_parser args_seq = p.eps(); diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 79b97a80f1b2..f827974952a3 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -318,13 +318,13 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) { bool is_content = node.tag == common_chat_peg_builder::CONTENT; if (is_reasoning) { // GPT OSS can have more than 1 reasoning block, so concatenate here - result.reasoning_content += std::string(node.text); + result.reasoning_content += node.sanitized_text(); } if (is_content) { // Concatenate content from multiple content nodes (e.g., when reasoning markers // are preserved before content markers in reasoning_format=NONE mode) - result.content += std::string(node.text); + result.content += node.sanitized_text(); } // Handle tool-related tags (supporting both JSON and tagged formats) @@ -488,7 +488,7 @@ common_peg_parser common_chat_peg_builder::standard_constructed_tools( } const auto & function = tool_def.at("function"); std::string name = function.at("name"); - ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object(); + ordered_json params = common_chat_tool_parameters(function); // Build argument parsers auto args = eps(); @@ -565,7 +565,7 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls( } const auto & function = tool_def.at("function"); std::string name = function.at("name"); - ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object(); + ordered_json params = common_chat_tool_parameters(function); auto args = eps(); if (params.contains("properties") && !params["properties"].empty()) { @@ -640,7 +640,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key( } const auto & function = tool_def.at("function"); std::string name = function.at("name"); - ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object(); + ordered_json params = common_chat_tool_parameters(function); // Build inner object fields std::vector<common_peg_parser> inner_fields; @@ -726,7 +726,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys( } const auto & function = tool_def.at("function"); std::string name = function.at("name"); - ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object(); + ordered_json params = common_chat_tool_parameters(function); auto nested_name = literal("\"" + nested_name_field + "\"") + space() + literal(":") + space() + atomic(literal("\"") + tool_name(literal(name)) + literal("\"")); @@ -795,7 +795,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys( } const auto & function = tool_def.at("function"); std::string name = function.at("name"); - ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object(); + ordered_json params = common_chat_tool_parameters(function); auto tool_name_ = name_key_parser + space() + literal(":") + space() + atomic(literal("\"") + tool_name(literal(name)) + literal("\"")); @@ -1058,12 +1058,12 @@ void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, co const auto & node = arena.get(id); if (node.tag == "reasoning") { - result.reasoning_content += std::string(node.text); + result.reasoning_content += node.sanitized_text(); return; } if (node.tag == "content") { - result.content += std::string(node.text); + result.content += node.sanitized_text(); return; } @@ -1206,12 +1206,12 @@ void common_chat_peg_minimax_m3_mapper::visit(const common_peg_ast_arena & arena const auto & node = arena.get(id); if (node.tag == common_chat_peg_builder::REASONING) { - result.reasoning_content += std::string(node.text); + result.reasoning_content += node.sanitized_text(); return; } if (node.tag == common_chat_peg_builder::CONTENT) { - result.content += std::string(node.text); + result.content += node.sanitized_text(); return; } diff --git a/common/chat.cpp b/common/chat.cpp index a961c8a784df..ed1942e15349 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -8,6 +8,7 @@ #include "json-schema-to-grammar.h" #include "json.h" #include "log.h" +#include "parsers/parsers.h" #include "jinja/value.h" #include "jinja/runtime.h" @@ -573,6 +574,16 @@ json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & t return result; } +json common_chat_tool_parameters(const json & function) { + if (function.contains("parameters")) { + const auto & params = function.at("parameters"); + if (!params.is_null() && !(params.is_object() && params.empty())) { + return params; + } + } + return json{{"type", "object"}, {"properties", json::object()}}; +} + std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & tools) { std::vector<common_chat_tool> result; @@ -717,13 +728,6 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp return tmpls->has_explicit_template; } -// LFM2 format detection: template uses <|tool_list_start|>[...]<|tool_list_end|> around the tool list -// and <|tool_call_start|>[...]<|tool_call_end|> around each tool call -static bool is_lfm2_template(const std::string & src) { - return src.find("<|tool_list_start|>") != std::string::npos && - src.find("<|tool_list_end|>") != std::string::npos; -} - common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) { common_chat_prompt_preset asr_preset; asr_preset.system = ""; @@ -898,42 +902,12 @@ common_reasoning_format common_reasoning_format_from_name(const std::string & fo throw std::runtime_error("Unknown reasoning format: " + format); } -static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) { - for (const auto & tool : tools) { - if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) { - LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str()); - continue; - } - fn(tool); - } -} - -static void foreach_parameter(const json & function, - const std::function<void(const std::string &, const json &, bool)> & fn) { - if (!function.contains("parameters") || !function.at("parameters").is_object()) { - return; - } - const auto & params = function.at("parameters"); - if (!params.contains("properties") || !params.at("properties").is_object()) { - return; - } - const auto & props = params.at("properties"); - std::set<std::string> required; - if (params.contains("required") && params.at("required").is_array()) { - required = params.at("required").get<std::set<std::string>>(); - } - for (const auto & [name, prop] : props.items()) { - bool is_required = (required.find(name) != required.end()); - fn(name, prop, is_required); - } -} - -static std::string common_chat_template_direct_apply_impl( +std::string common_chat_template_direct_apply_impl( const common_chat_template & tmpl, const autoparser::generation_params & inputs, - const std::optional<json> & messages_override = std::nullopt, - const std::optional<json> & tools_override = std::nullopt, - const std::optional<json> & additional_context = std::nullopt) { + const std::optional<json> & messages_override, + const std::optional<json> & tools_override, + const std::optional<json> & additional_context) { jinja::context ctx(tmpl.source()); // messages_override is already built for this template, do not touch its content parts @@ -997,12 +971,12 @@ std::string common_chat_template_direct_apply( return common_chat_template_direct_apply_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt); } -static std::string common_chat_template_generation_prompt_impl( +std::string common_chat_template_generation_prompt_impl( const common_chat_template & tmpl, const autoparser::generation_params & inputs, - const std::optional<json> & messages_override = std::nullopt, - const std::optional<json> & tools_override = std::nullopt, - const std::optional<json> & additional_context = std::nullopt) { + const std::optional<json> & messages_override, + const std::optional<json> & tools_override, + const std::optional<json> & additional_context) { autoparser::generation_params params = inputs; params.add_generation_prompt = false; @@ -1025,2521 +999,146 @@ std::string common_chat_template_generation_prompt( return common_chat_template_generation_prompt_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt); } -static common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - // Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja - auto adjusted_messages = json::array(); - for (const auto & msg : inputs.messages) { - auto role = msg.value("role", ""); - if (role != "system" && role != "assistant") { - // Only adjust system and assistant messages. Interestingly, the system message may contain thinking. - adjusted_messages.push_back(msg); - continue; - } - - auto content = json::array(); - - // If message contains `reasoning_content`, add it as a block of type `thinking` - if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { - content.push_back({ - { "type", "thinking" }, - { "thinking", msg.at("reasoning_content").get<std::string>() }, - }); - } +namespace workaround { - // If message contains `content`, add it as a block of type `text` - if (msg.contains("content")) { - if (msg.at("content").is_string()) { - content.push_back({ - { "type", "text" }, - { "text", msg.at("content").get<std::string>() }, - }); - } else if (msg.at("content").is_array()) { - auto blocks = msg.at("content"); - content.insert(blocks); +static void map_developer_role_to_system(json & messages) { + for (auto & message : messages) { + if (message.contains("role")) { + if (message["role"] == "developer") { + message["role"] = "system"; } } - - auto adjusted = msg; - adjusted["content"] = content; - adjusted.erase("reasoning_content"); - adjusted_messages.push_back(adjusted); - } - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = true; - - data.supports_thinking = true; - data.thinking_start_tag = "[THINK]"; - data.thinking_end_tags = {"[/THINK]"}; - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.preserved_tokens = { - "[THINK]", - "[/THINK]", - "[TOOL_CALLS]", - "[ARGS]", - }; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = "[THINK]" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "[/THINK]" + msg.render_content(); - } - - data.prompt += data.generation_prompt; } +} - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.eps(); - auto reasoning = - extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps(); - - // Response format parser - if (has_response_format) { - // Ministral wants to emit json surrounded by code fences - return generation_prompt + (reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```"); - } - - // Tool call parser - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & schema = function.at("parameters"); - - tool_choice |= - p.rule("tool-" + name, p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") + - p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))); - }); - - auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; - auto max_calls = inputs.parallel_tool_calls ? -1 : 1; - auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls)); - return generation_prompt + (reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls); +// if first message is system and template does not support it, merge it with next message +static void system_message_not_supported(json & messages) { + if (!messages.empty() && messages.front().at("role") == "system") { + if (messages.size() > 1) { + LOG_DBG("Merging system prompt into next message\n"); + auto & first_msg = messages.front(); + auto & second_msg = messages[1]; + second_msg["content"] = first_msg.at("content").get<std::string>() + + "\n" + second_msg.at("content").get<std::string>(); + messages.erase(0); + } else { + LOG_WRN("Removing system prompt due to template not supporting system role\n"); + messages.erase(0); } - - // Content only parser - include_grammar = false; - return generation_prompt + (reasoning << p.content(p.rest())); - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]" } - }; } - - return data; } -static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - const std::string GEN_PREFIX = "<|im_start|>assistant\n"; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - - auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos; - - data.supports_thinking = supports_reasoning; - data.preserved_tokens = { - "<tool_call>", - "</tool_call>", - }; - - auto is_qwen3_coder = !supports_reasoning; - - if (supports_reasoning) { - data.thinking_start_tag = "<think>"; - // Support both </think> and <tool_call> as reasoning end sequences. - // <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model - data.thinking_end_tags = { "</think>", "<tool_call>" }; - data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" }); - } - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3 - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash - { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = GEN_PREFIX; - if (supports_reasoning) { - data.generation_prompt += "<think>\n" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "\n</think>\n\n"; - } - } - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += msg.render_content(); +static void requires_non_null_content(json & messages) { + GGML_ASSERT(messages.is_array()); + for (auto & message : messages) { + if (message.contains("tool_calls") && !message.contains("content")) { + message["content"] = ""; } - - data.prompt += data.generation_prompt; - } - - std::vector<std::string> tool_call_starts = { "<tool_call>" }; - - if (is_qwen3_coder) { - // Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the - // starting <tool_call>. The model may hallucinate a tool name, but it is preferable over - // constraining on <function which may occur in valid content generation, e.g. #include <functional> - foreach_function(inputs.tools, [&](const json & tool) { - const std::string name = tool.at("function").at("name"); - tool_call_starts.push_back("<function=" + name + ">"); - }); } +} - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal(GEN_PREFIX); - - auto reasoning = p.eps(); - if (supports_reasoning && extract_reasoning) { - reasoning = p.optional("<think>" + p.space() + - p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) + - (p.literal("</think>") | p.peek(p.literal("<tool_call>")))); - } - - // Response format parser - if (has_response_format) { - return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema))); - } - - // Tool call parser - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto arg_close = p.tool_arg_close(p.literal("\n</parameter>\n")); - auto arg_string = p.rule("xml-arg-string", - p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n")); - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - auto parameters = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(parameters); - - std::vector<common_peg_parser> required_args; - std::vector<common_peg_parser> optional_args; - - foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) { - auto rule_name = "tool-" + name + "-arg-" + param_name; - - auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param_name)) + ">\n"); - - auto arg_value = schema_info.resolves_to_string(param_schema) ? - arg_string : - p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close; - - auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); - - (is_required ? required_args : optional_args).push_back(arg_rule); - }); - - // Accept required arguments in any order, as Qwen does not always adhere to the - // order provided. - auto args = p.permute("tool-" + name + "-args", required_args); - if (!optional_args.empty()) { - args = args + p.zero_or_more(p.choice(optional_args)); +static void func_args_not_string(json & messages) { + GGML_ASSERT(messages.is_array()); + for (auto & message : messages) { + if (message.contains("tool_calls")) { + for (auto & tool_call : message["tool_calls"]) { + if (tool_call.contains("function") && tool_call["function"].contains("arguments")) { + auto & args = tool_call["function"]["arguments"]; + if (args.is_string()) { + try { + args = json::parse(args.get<std::string>()); + } catch (const std::exception & e) { + throw std::runtime_error("Failed to parse tool call arguments as JSON: " + std::string(e.what())); + } + } } - - auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") + - p.tool_args(args) + - p.tool_close(p.literal("</function>\n"))); - - tool_choice |= p.rule("tool-" + name, func); - }); - - auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; - - auto tool_call_body = tool_choice + "</tool_call>" + p.space(); - auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body); - - // Qwen3-Coder models may occasionally omit the <tool_call> token. - auto tool_call_first = is_qwen3_coder ? - p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body) : - tool_call; - - auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first; - auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1)); - - return generation_prompt + - (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls); - } - - // Content only parser - return generation_prompt + (reasoning << p.content(p.rest())); - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - if (data.grammar_lazy) { - for (const auto & start : tool_call_starts) { - data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start }); } } } - - return data; } -static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - // Copy reasoning to the "thinking" field as expected by the gpt-oss template - auto adjusted_messages = json::array(); - for (auto msg : inputs.messages) { - if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { - msg["thinking"] = msg.at("reasoning_content"); - if (msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) { - msg.erase("content"); +// Trim leading/trailing whitespace from message contents before rendering. This +// has to run on the messages (not on the rendered JSON) because templates with +// string-only content caps concatenate typed content parts into a single string +// during rendering, after which the per-part whitespace can no longer be reached. +// Both the plain string content and the text of typed content parts are trimmed. +static void trim_all_content(std::vector<common_chat_msg> & messages) { + for (auto & message : messages) { + message.content = trim_whitespace(message.content); + message.reasoning_content = trim_whitespace(message.reasoning_content); + for (auto & part : message.content_parts) { + if (part.type == "text") { + part.text = trim_whitespace(part.text); } } - adjusted_messages.push_back(msg); } +} - auto prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override= */ adjusted_messages); - - // Check if we need to replace the return token with end token during - // inference and without generation prompt. For more details see: - // https://github.com/ggml-org/llama.cpp/issues/15417 - if (inputs.is_inference && !inputs.add_generation_prompt) { - static constexpr std::string_view return_token = "<|return|>"; - static constexpr std::string_view end_token = "<|end|>"; - if (size_t pos = prompt.rfind(return_token); pos != std::string::npos) { - prompt.replace(pos, return_token.length(), end_token); - } - } - - data.prompt = prompt; - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages); - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, - { COMMON_CHAT_ROLE_USER, "<|start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, - { COMMON_CHAT_ROLE_TOOL, "<|start|>functions" }, - }; - - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - - data.thinking_start_tag = "<|channel|>analysis<|message|>"; - data.thinking_end_tags = {"<|end|>"}; - - // These special tokens are required to parse properly, so we include them - // even if parse_tool_calls is false. - data.preserved_tokens = { - "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>", - }; - - // Adjust prompt for continuation - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; +} - data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content(); - } +static json common_chat_extra_context() { + json ctx = json::object(); + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + std::string datetime_str = format_time(now, "%b %d %Y"); + std::string date_str = format_time(now, "%d %b %Y"); + ctx["datetime"] = datetime_str; + ctx["date_string"] = date_str; + return ctx; +} - data.prompt += data.generation_prompt; +std::optional<common_chat_params> common_chat_try_specialized_template( + const common_chat_template & tmpl, + const std::string & src, + autoparser::generation_params & params) { + // Ministral/Mistral Large 3 - uses special reasoning structure fixes, can't use autoparser + // Note: Mistral Small 3.2 uses [CALL_ID] which Ministral doesn't have, so we can distinguish them + if (src.find("[SYSTEM_PROMPT]") != std::string::npos && src.find("[TOOL_CALLS]") != std::string::npos && + src.find("[ARGS]") != std::string::npos && src.find("[CALL_ID]") == std::string::npos) { + LOG_DBG("Using specialized template: Ministral/Magistral Large 3\n"); + return common_chat_params_init_ministral_3(tmpl, params); } - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto start = p.rule("start", p.literal("<|start|>assistant")); - auto end = p.rule("end", p.literal("<|end|>")); - auto content = p.rule("message-content", p.until("<|end|>")); - auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis")); - auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1); - - // Occasionally, gpt-oss-20b will prefix channels with this commentary - auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant"))); - auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>"); - - if (extract_reasoning) { - p.rule("analysis", start_analysis + p.reasoning(content) + end); - } else { - p.rule("analysis", p.content(start_analysis + content + end)); - } - - auto analysis = p.ref("analysis"); - auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end); - auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content)); - - // Consume any unsolicited tool calls, e.g. builtin functions - auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end)); - - auto any = p.rule("any", preamble | analysis); - - if (has_response_format) { - auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); - auto response_format = p.rule("response-format", - p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + - p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema))); - - return p.zero_or_more(start + analysis) + start + response_format; - } - - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto tool_choice = p.choice(); - - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & params = function.at("parameters"); - - auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name)); - auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); - auto args = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", params)); - - // recipient in role header - // <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS - auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args); - - // recipient in channel header - // <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS - auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args); - - tool_choice |= p.rule("tool-" + name, tool_in_role | tool_in_channel); - }); - - auto tool_call = p.trigger_rule("tool-call", tool_choice); - - if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { - return p.zero_or_more(start + any) + start + tool_call; - } - - return p.zero_or_more(start + any) + start + (tool_call | final_msg); - } - - return p.zero_or_more(start + any) + start + (final_msg | unsolicited); - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" }, - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" }, - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" }, - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" } - }; + // GPT-OSS - has unique channel-based structure that needs dedicated handler + if (src.find("<|channel|>") != std::string::npos) { + LOG_DBG("Using specialized template: GPT-OSS\n"); + return common_chat_params_init_gpt_oss(tmpl, params); } - return data; -} - -static common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - - if (inputs.add_generation_prompt && string_ends_with(data.prompt, "<turn|>\n")) { - // This may happen if the model generates content + tool_call, the - // template does not add the model's next turn and confuses the model - // from emitting its proper reasoning token sequence. - data.generation_prompt = "<|turn>model\n"; - data.prompt += data.generation_prompt; + // Muse Glimmer format using " to=<recipient>" recipients and <|eom|>/<|eot|> message terminators. + if (src.find("<atem:function_calls>") != std::string::npos && src.find("<|eom|>") != std::string::npos) { + LOG_DBG("Using specialized template: Muse Glimmer\n"); + return common_chat_params_init_muse_glimmer(tmpl, params); } - data.message_delimiters = { - { COMMON_CHAT_ROLE_USER, "<|turn>user" }, - { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" }, - }; - - data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4; - data.supports_thinking = true; - data.thinking_start_tag = "<|channel>thought"; - data.thinking_end_tags = {"<channel|>"}; - - data.preserved_tokens = { - "<|channel>", - "<channel|>", - "<|tool_call>", - "<tool_call|>", - "<|turn>", - }; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = string_ends_with(data.prompt, "<turn|>\n") ? "<|turn>model\n" : ""; - data.generation_prompt += "<|channel>thought\n" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "<channel|>" + msg.render_content(); - } - - data.prompt += data.generation_prompt; + // Functionary v3.2 - uses recipient-based format with >>>recipient\n{content} + // Detection: template has ">>>all" for content and ">>>" prefix for tool calls + if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) { + LOG_DBG("Using specialized template: Functionary v3.2\n"); + return common_chat_params_init_functionary_v3_2(tmpl, params); } - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto start = p.rule("start", p.optional(p.literal("<|turn>model\n"))); - - if (extract_reasoning) { - p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>")); - } else { - p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("<channel|>") + p.literal("<channel|>"))); - } + // Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index> + // Detection: template has "<|tool_calls_section_begin|>" and "functions." prefix in tool call IDs + if (src.find("<|tool_calls_section_begin|>") != std::string::npos && + src.find("<|tool_call_begin|>") != std::string::npos) { + LOG_DBG("Using specialized template: Kimi K2 Thinking\n"); + return common_chat_params_init_kimi_k2(tmpl, params); + } - auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), ""); - auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>")); + // Kimi K3 - the <|open|>/<|close|>/<|end_of_msg|> markers are unique to it + if (src.find("<|open|>") != std::string::npos && src.find("<|close|>") != std::string::npos && + src.find("<|end_of_msg|>") != std::string::npos) { + LOG_DBG("Using specialized template: Kimi K3\n"); + return common_chat_params_init_kimi_k3(tmpl, params); + } - if (has_response_format) { - auto response_format = p.literal("```json") << - p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) << - p.literal("```"); - return start + p.optional(thought) + response_format; - } - - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - // Gemma4 tool calling syntax - // Rules should match traversal logic in gemma4_to_json() - p.rule("gemma4-string-content", p.until("<|\"|>")); - p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>")); - p.rule("gemma4-bool", p.json_bool()); - p.rule("gemma4-null", p.json_null()); - p.rule("gemma4-number", p.json_number()); - p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":")); - p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value")); - p.rule("gemma4-dict", [&]() { - auto ws = p.space(); - auto member = p.ref("gemma4-dict-kv"); - auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))}); - return p.sequence({ - p.literal("{"), ws, - p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})}) - }); - }); - p.rule("gemma4-array", [&]() { - auto ws = p.space(); - auto value = p.ref("gemma4-value"); - auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))}); - return p.sequence({ - p.literal("["), ws, - p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})}) - }); - }); - p.rule("gemma4-value", [&]() { - return p.choice({ - p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"), - p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null") - }); - }); - - auto tool_choice = p.choice(); - - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules - // const auto & params = function.at("parameters"); - - tool_choice |= p.rule("tool-" + name, p.tool(p.sequence({ - p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))), - p.tool_args(p.ref("gemma4-dict")), - }))); - }); - - auto tool_call = p.trigger_rule("tool-call", p.repeat( - "<|tool_call>call:" + tool_choice + "<tool_call|>", - /* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0, - /* max = */ inputs.parallel_tool_calls ? -1 : 1 - )); - - auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>")); - auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>", "<|tool_call>"}))); - auto message = p.rule("message", thought + content); - return start + p.zero_or_more(message) + scan_to_toolcall + tool_call; - } - - // Gemma 4 may emit an extra <|channel>thought\n<channel|> at the end of the content. It may - // also emit a single trailing <channel|> token. Consume all complete reasoning blocks and - // then stop at the first unmatched <channel|> token. - auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>"}))); - auto message = p.rule("message", thought + content); - return start + p.one_or_more(message); - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" }, - }; - } - - return data; -} - -// Functionary v3.2 - uses recipient-based format: >>>recipient\n{content} -static common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.preserved_tokens = { - ">>>all", - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - data.generation_prompt = "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\n" + msg.render_content(); - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - // Functionary v3.2 format: - // - Normal content: >>>all\n{content} - // - Tool calls: >>>function_name\n{json_args} - // Generation prompt ends with ">>>" so model outputs recipient immediately - - // Build content parser for >>>all\n{content} - // When tools are present, content stops before the next ">>>" (tool call) - // When no tools, content goes until end - auto content_until_tool = p.literal("all\n") + p.content(p.until(">>>")); - auto content_until_end = p.literal("all\n") + p.content(p.rest()); - auto generation_prompt = p.literal("<|start_header_id|>assistant<|end_header_id|>\n\n>>>"); - - // If no tools or tool_choice is NONE, just parse content - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - // When no tools, just match the prefix and capture everything after - return generation_prompt + content_until_end + p.end(); - } - - // Build tool call parsers for each available function - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & schema = function.at("parameters"); - - // Tool format: >>>function_name\n{json_args} - auto tool_parser = p.tool( - p.tool_open(p.tool_name(p.literal(name)) + p.literal("\n")) + - p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) - ); - - tool_choice |= p.rule("tool-" + name, tool_parser); - }); - - auto content_only = content_until_end; - auto tools_only = p.trigger_rule("tools", p.one_or_more(tool_choice)); - auto content_and_tools = content_until_tool + tools_only; - - auto ret = p.eps(); - if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { - if (inputs.parallel_tool_calls) { - ret = p.choice({ content_and_tools, tools_only }) + p.end(); - } else { - ret = p.choice({ content_until_tool + tool_choice, tools_only }) + p.end(); - } - } else if (inputs.parallel_tool_calls) { - ret = p.choice({ content_and_tools, content_only, tools_only }) + p.end(); - } else { - auto content_and_tool = content_until_tool + tool_choice; - ret = p.choice({ content_and_tool, content_only, tool_choice }) + p.end(); - } - return generation_prompt + ret; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - parser.build_grammar(builder, data.grammar_lazy); - }); - - // Grammar trigger for when the model starts outputting a tool call - // (after the initial ">>>" in the generation prompt but recipient other than "all") - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, ">>>(?!all)" } - }; - } - - return data; -} - -// Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index> -// The ID contains both the function name and an incrementing counter -static common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.preserved_tokens = { - "<|tool_calls_section_begin|>", - "<|tool_calls_section_end|>", - "<|tool_call_begin|>", - "<|tool_call_argument_begin|>", - "<|tool_call_end|>", - "<think>", - "</think>", - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - - const std::string SECTION_BEGIN = "<|tool_calls_section_begin|>"; - const std::string SECTION_END = "<|tool_calls_section_end|>"; - const std::string CALL_BEGIN = "<|tool_call_begin|>"; - const std::string ARGS_BEGIN = "<|tool_call_argument_begin|>"; - const std::string CALL_END = "<|tool_call_end|>"; - - const std::string THINK_START = "<think>"; - const std::string THINK_END = "</think>"; - const std::string GEN_PROMPT = "<|im_assistant|>assistant<|im_middle|>"; - - data.thinking_start_tag = THINK_START; - data.thinking_end_tags = {THINK_END}; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - // Kimi K2 Thinking format: - // - Reasoning: <think>{reasoning}</think> - // - Content: text after reasoning - // - Tool calls section: - // <|tool_calls_section_begin|> - // <|tool_call_begin|>functions.<name>:<index><|tool_call_argument_begin|>{json_args}<|tool_call_end|> - // ... - // <|tool_calls_section_end|> - // The ID format is: functions.<function_name>:<counter> where counter is 0, 1, 2, ... - - // Tool call markers - auto end = p.end(); - - // Note: this model is CRAZY. It can diverge from its supposed tool calling pattern in so many ways it's not funny. - // For example, it can call tools at the end of reasoning without closing reasoning... - auto reasoning = extract_reasoning ? p.optional(THINK_START + p.reasoning( - p.until_one_of({ THINK_END, "<|tool_calls_section_begin|>", "<|tool_call_begin|>" })) + - p.optional(p.literal(THINK_END))) : p.eps(); - auto generation_prompt = p.literal(GEN_PROMPT); - - - // Content only parser (no tools) - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - return generation_prompt + reasoning + p.content(p.rest()) + end; - } - - // Build tool call parsers for each available function - // The ID format is: functions.<name>:<index> - // We need to match: functions.<name>:<digits> - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & schema = function.at("parameters"); - - // Match: functions.<name>:<digits> - // Capture the full call id (functions.<name>:<digits>) using tool_id tag - auto tool_id = p.tool_id(p.literal("functions.") + p.tool_name(p.literal(name)) + p.literal(":") + p.chars("[0-9]", 1, -1)); - auto tool_parser = p.tool( - p.tool_open(tool_id + p.literal(ARGS_BEGIN)) + - p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) + - p.tool_close(p.optional((p.literal(CALL_END)))) - ); - - tool_choice |= p.rule("tool-" + name, tool_parser); - }); - - // Tool calls section: <|tool_calls_section_begin|> tool_calls <|tool_calls_section_end|> - auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; - auto max_calls = inputs.parallel_tool_calls ? -1 : 1; - // Use trigger_rule so grammar generator knows where to start generating rules - auto tool_calls = p.rule("tool-calls", - p.optional(p.literal(SECTION_BEGIN)) + - p.trigger_rule("tool-call", p.repeat(CALL_BEGIN + tool_choice, min_calls, max_calls) + - p.optional(p.literal(SECTION_END))) - ); - - auto content_before_tools = p.content(p.until_one_of({ SECTION_BEGIN, CALL_BEGIN })); - - return generation_prompt + reasoning + content_before_tools + tool_calls + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call_begin|>" } - }; - } - - return data; -} - -// LFM2/LFM2.5 parser. Tool calls are almost Python-style and parallel-capable -// (except dotted names and JSON literals true/false/null). -// Always wrapped in <|tool_call_start|>[name(args)]<|tool_call_end|> with optional <think> reasoning. -// tool_list_tokens preserves LFM2 system tool-list markers. -static common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl, - const autoparser::generation_params & inputs, - bool tool_list_tokens) { - common_chat_params data; - - const std::string TOOL_CALL_START = "<|tool_call_start|>"; - const std::string TOOL_CALL_END = "<|tool_call_end|>"; - const std::string TOOL_LIST_START = "<|tool_list_start|>"; - const std::string TOOL_LIST_END = "<|tool_list_end|>"; - const std::string THINK_START = "<think>"; - const std::string THINK_END = "</think>"; - const std::string GEN_PROMPT = "<|im_start|>assistant\n"; - - // Copy reasoning to the "thinking" field the template expects - auto adjusted_messages = json::array(); - for (auto msg : inputs.messages) { - if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { - msg["thinking"] = msg.at("reasoning_content"); - } - adjusted_messages.push_back(msg); - } - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.preserved_tokens = { TOOL_CALL_START, TOOL_CALL_END, THINK_START, THINK_END }; - if (tool_list_tokens) { - data.preserved_tokens.push_back(TOOL_LIST_START); - data.preserved_tokens.push_back(TOOL_LIST_END); - } - - data.thinking_start_tag = THINK_START; - data.thinking_end_tags = {THINK_END}; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); - // Gate by reasoning format and whether the template supports <think> - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE && - tmpl.source().find(THINK_START) != std::string::npos; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal(GEN_PROMPT); - auto end = p.end(); - - auto reasoning = p.eps(); - if (extract_reasoning) { - // LFM2.5's template does not prefill an opening <think> into the generation prompt, - // so the model must emit it itself each turn. It sometimes skips the opening tag and - // dives straight into reasoning, closing only with </think>. Without forced-open - // handling that raw reasoning bleeds into content. Make the opening tag - // optional so a leading reasoning block terminated by </think> is captured whether or - // not <think> was emitted. A pure-content turn has no </think>, so until() fails to - // find the terminator, the sequence fails, and the outer optional collapses -> the - // whole output falls through to content exactly as before. - reasoning = p.optional(p.optional(p.literal(THINK_START)) + p.reasoning(p.until(THINK_END)) + p.literal(THINK_END)); - } - - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - if (has_response_format) { - auto response_format = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)); - return generation_prompt + reasoning + response_format + end; - } - return generation_prompt + reasoning + p.content(p.rest()) + end; - } - auto tool_calls = p.rule("tool-calls", - p.trigger_rule("tool-call", - p.literal(TOOL_CALL_START) + - p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true) + - p.literal(TOOL_CALL_END) - ) - ); - - auto content = p.content(p.until(TOOL_CALL_START)); - - return generation_prompt + reasoning + content + tool_calls + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOL_CALL_START } - }; - } - - return data; -} - -static common_chat_params common_chat_params_init_gigachat_v3( - const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = false; - data.preserved_tokens = { - "<|message_sep|>\n\n", - "<|role_sep|>\n", - }; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - data.generation_prompt = "assistant<|role_sep|>\n" + msg.render_content(); - data.prompt += data.generation_prompt; - } - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n"; - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto ret = p.eps(); - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - // Build a choice of all available tools - auto tool_choice = p.choice(); - for (const auto & tool : inputs.tools) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & schema = function.at("parameters"); - - auto tool_name = p.json_member("name", "\"" + p.tool_name(p.literal(name)) + "\""); - auto tool_args = p.json_member("arguments", p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))); - - auto tool_open = p.tool_open(p.literal("{") << tool_name); - - tool_choice |= p.rule("tool-" + name, tool_open << "," << tool_args << "}"); - } - - // Define the tool call structure - auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; - auto max_calls = 1; // parallel toolcalls are not supported - auto tool_call = p.rule("tool-call", p.literal(tool_call_start_prefix) + tool_choice); - auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(tool_call, /* min = */ min_calls, /* max = */ max_calls)); - - ret = p.content(p.until("<|message_sep|>\n\n")) << tool_calls; - } else { - // Content only parser - include_grammar = false; - ret = p.content(p.rest()); - } - - return p.literal("assistant<|role_sep|>\n") + ret; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - {COMMON_GRAMMAR_TRIGGER_TYPE_WORD, tool_call_start_prefix} - }; - } - return data; -} - -// The DeepSeek V4 reference implementation renders consecutive tool results into a single -// user block, ordered by the tool call order of the preceding assistant message (matched -// by tool call id) rather than by the order they appear in the conversation. -static json deepseek_v4_sort_tool_results(const json & messages) { - json adjusted = messages; - std::map<std::string, size_t> call_order; - - for (size_t i = 0; i < adjusted.size();) { - const auto & msg = adjusted[i]; - const auto role = msg.value("role", ""); - - if (role == "assistant" && msg.contains("tool_calls") && - msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) { - call_order.clear(); - const auto & tool_calls = msg.at("tool_calls"); - for (size_t idx = 0; idx < tool_calls.size(); idx++) { - auto id = tool_calls[idx].value("id", ""); - if (!id.empty()) { - call_order[id] = idx; - } - } - i++; - continue; - } - - if (role != "user" && role != "tool") { - i++; - continue; - } - - // collect a maximal run of user/tool messages - they render into one user block - std::vector<size_t> tool_positions; - size_t run_end = i; - for (; run_end < adjusted.size(); run_end++) { - const auto r = adjusted[run_end].value("role", ""); - if (r == "tool") { - tool_positions.push_back(run_end); - } else if (r != "user") { - break; - } - } - - if (tool_positions.size() > 1 && !call_order.empty()) { - std::vector<json> results; - results.reserve(tool_positions.size()); - for (auto pos : tool_positions) { - results.push_back(adjusted[pos]); - } - std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) { - const auto order = [&](const json & m) { - auto it = call_order.find(m.value("tool_call_id", "")); - return it == call_order.end() ? (size_t) 0 : it->second; - }; - return order(a) < order(b); - }); - for (size_t k = 0; k < tool_positions.size(); k++) { - adjusted[tool_positions[k]] = std::move(results[k]); - } - } - - i = run_end; - } - - return adjusted; -} - -static common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - // V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls" - // instead of "function_calls", renders tool results in tool call order and its - // non-thinking generation prompt ends with a bare </think> instead of an empty - // <think></think> pair. - const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos; - // V4.1 keeps the V4 format but its DSML tag names carry a leading space: - // "<｜DSML｜ calls>", "<｜DSML｜ invoke>", "<｜DSML｜ parameter>" (the release's - // encoding/encoding.py: tool_calls_block_name = " calls"). The template builds the - // opener as "<' + dsml_token + ' calls>" (single or double quotes), so that literal - // identifies a V4.1 template. - const bool is_v41 = is_v4 && (tmpl.source().find("' calls>") != std::string::npos || - tmpl.source().find("\" calls>") != std::string::npos); - - std::optional<json> adjusted_messages; - if (is_v4) { - adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages); - } - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - std::optional<json> additional_context; - if (is_v4 && has_response_format) { - additional_context = json{ { "response_format", inputs.json_schema } }; - } - - const std::string DSML = "｜DSML｜"; - const std::string THINK_START = "<think>"; - const std::string THINK_END = "</think>"; - const std::string TC_BLOCK = is_v41 ? " calls" : is_v4 ? "tool_calls" : "function_calls"; - const std::string INVOKE_TAG = is_v41 ? " invoke" : "invoke"; - const std::string PARAM_TAG = is_v41 ? " parameter" : "parameter"; - const std::string FC_START = "<" + DSML + TC_BLOCK + ">"; - const std::string FC_END = "</" + DSML + TC_BLOCK + ">"; - const std::string INVOKE_START = "<" + DSML + INVOKE_TAG; - const std::string INVOKE_END = "</" + DSML + INVOKE_TAG + ">"; - const std::string PARAM_START = "<" + DSML + PARAM_TAG; - const std::string PARAM_END = "</" + DSML + PARAM_TAG + ">"; - const std::string GEN_PROMPT = "<｜Assistant｜>"; - const std::string TC_SEPARATOR = "\n\n"; - - data.prompt = common_chat_template_direct_apply_impl( - tmpl, inputs, adjusted_messages, std::nullopt, additional_context); - data.generation_prompt = common_chat_template_generation_prompt_impl( - tmpl, inputs, adjusted_messages, std::nullopt, additional_context); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.thinking_start_tag = THINK_START; - data.thinking_end_tags = {THINK_END, FC_START}; - data.preserved_tokens = { - DSML, - THINK_START, - THINK_END, - }; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - if (is_v4 && msg.reasoning_content.empty()) { - data.generation_prompt = GEN_PROMPT + THINK_END; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += msg.render_content(); - } - } else { - data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + msg.render_content(); - } - } - - data.prompt += data.generation_prompt; - } - - bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; - bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal(GEN_PROMPT); - auto end = p.end(); - - // build tool call section first since we might need it in reasoning - auto tool_choice = p.choice(); - if (has_tool_calls) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - const auto & props = params.contains("properties") ? params.at("properties") : json::object(); - - std::set<std::string> required; - if (params.contains("required")) { - required = params.at("required").get<std::set<std::string>>(); - } - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - std::vector<common_peg_parser> required_parsers; - std::vector<common_peg_parser> optional_parsers; - for (const auto & [param_name, param_schema] : props.items()) { - bool is_required = required.find(param_name) != required.end(); - bool is_string = schema_info.resolves_to_string(param_schema); - - auto arg = p.tool_arg( - p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) + - p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) + - (is_string ? - p.tool_arg_string_value(p.until(PARAM_END)) : - p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema", - param_schema, false))) + - p.tool_arg_close(p.literal(PARAM_END))); - - auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); - if (is_required) { - required_parsers.push_back(named_arg); - } else { - optional_parsers.push_back(named_arg); - } - } - - common_peg_parser args_seq = p.eps(); - for (size_t i = 0; i < required_parsers.size(); i++) { - if (i > 0) { - args_seq = args_seq + p.space(); - } - args_seq = args_seq + required_parsers[i]; - } - - if (!optional_parsers.empty()) { - common_peg_parser any_opt = p.choice(); - for (const auto & opt : optional_parsers) { - any_opt |= opt; - } - args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1); - } - - common_peg_parser invoke_body = args_seq; - auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") + - p.tool_name(p.literal(name)) + p.literal("\">\n")) + - invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END))); - - tool_choice |= p.rule("tool-" + name, func_parser); - }); - } - - common_peg_parser tool_calls = p.eps(); - if (inputs.parallel_tool_calls) { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + - p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); - } else { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); - } - - auto reasoning = p.eps(); - auto reasoning_with_tc = p.eps(); - auto obligatory_tool_calls = tool_calls; - bool allow_reasoning_with_tc = false; - - if (!require_tools) { - tool_calls = p.optional(tool_calls); - } - - if (extract_reasoning && inputs.enable_thinking) { - reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); - reasoning_with_tc = THINK_START + - p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) + - p.space() + obligatory_tool_calls; - allow_reasoning_with_tc = true; - } else if (extract_reasoning) { - // Thinking disabled but reasoning extraction requested: the generation prompt - // contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that - // must still be consumed. - reasoning = is_v4 - ? p.optional(p.literal(THINK_END)) - : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END)); - } - - if (has_response_format) { - auto response_format = p.rule("response-format", - p.literal("```json") + p.space() + - p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + - p.space() + p.literal("```")); - return generation_prompt + reasoning + response_format + end; - } - - if (!has_tool_calls) { - return generation_prompt + reasoning + p.content(p.rest()) + end; - } - - auto content_before_tools = p.negate(p.literal(THINK_START)) + - p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) + - p.space(); - return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end : - generation_prompt + reasoning + content_before_tools + tool_calls + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = has_tools && !require_tools; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START }, - }; - } - - return data; -} - -// Kimi K3 - XTML tagged format, built by open_tag/close_tag macros: -// open_tag(t, attrs) = <|open|>t k="v"...<|sep|> close_tag(t) = <|close|>t<|sep|> -// assistant := [think] [response] [tools] close_tag(message) <|end_of_msg|> -// the generation prompt already opens the think (or response) section, so the -// section opener is optional here - same as Kimi K2 Thinking -static common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - - const std::string SEP = "<|sep|>"; - const std::string MSG_START = "<|open|>message role=\"assistant\"<|sep|>"; - const std::string THINK_START = "<|open|>think<|sep|>"; - const std::string THINK_END = "<|close|>think<|sep|>"; - const std::string RESP_START = "<|open|>response<|sep|>"; - const std::string RESP_END = "<|close|>response<|sep|>"; - const std::string TOOLS_START = "<|open|>tools<|sep|>"; - const std::string TOOLS_END = "<|close|>tools<|sep|>"; - const std::string CALL_START = "<|open|>call tool=\""; - const std::string CALL_END = "<|close|>call<|sep|>"; - const std::string ARG_START = "<|open|>argument key=\""; - const std::string ARG_END = "<|close|>argument<|sep|>"; - const std::string MSG_END = "<|close|>message<|sep|>"; - const std::string EOM_TOKEN = "<|end_of_msg|>"; - - // only the markers are special tokens. tag names ("think", "response", ...) are - // normal tokens and must not be preserved, or prose with those words is broken - data.preserved_tokens = { - "<|open|>", - "<|close|>", - "<|sep|>", - "<|end_of_msg|>", - }; - - data.thinking_start_tag = THINK_START; - data.thinking_end_tags = { THINK_END }; - - // per-role message-start delimiters. user/assistant messages only have the role - // attribute, so the full opener is used. system and tool messages have more - // attributes, so those delimiters stop after the closing quote of the role - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|open|>message role=\"assistant\"<|sep|>" }, - { COMMON_CHAT_ROLE_USER, "<|open|>message role=\"user\"<|sep|>" }, - { COMMON_CHAT_ROLE_TOOL, "<|open|>message role=\"tool\"" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|open|>message role=\"system\"" }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = MSG_START + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + RESP_START + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto end = p.end(); - - auto start = p.optional(p.literal(MSG_START)); - - // the think section is always consumed, even with reasoning extraction off: - // the generation prompt ends with open_tag('think'), so it is always present. - // reasoning stops at its own closer, or at the response opener if the model - // skips the closer - auto think_body = extract_reasoning ? p.reasoning(p.until_one_of({ THINK_END, RESP_START })) : - p.content(p.until_one_of({ THINK_END, RESP_START })); - - auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body + - p.optional(p.literal(THINK_END))); - - // content runs to the response closer, or to the next section if truncated - auto response = p.optional(p.literal(RESP_START)) + - p.content(p.until_one_of({ RESP_END, TOOLS_START, MSG_END })) + - p.optional(p.literal(RESP_END)); - - // the EOG token after the message closer reaches the parser as text, - // so it must be consumed or the parse stays incomplete - auto trailer = p.optional(p.literal(MSG_END)) + p.optional(p.literal(EOM_TOKEN)); - - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - return start + reasoning + response + trailer + end; - } - - auto tool_choices = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const json schema = function.contains("parameters") ? function.at("parameters") : json::object(); - - // arguments come one tag per key, with the JSON type in a type="..." - // attribute. the type is taken from the tool schema instead, as it tells - // us if the value is JSON or a literal string - auto args = p.eps(); - if (schema.contains("properties") && !schema.at("properties").empty()) { - auto arg_choices = p.choice(); - for (const auto & prop : schema.at("properties").items()) { - const std::string & key = prop.key(); - - std::string type = "string"; - if (prop.value().is_object() && prop.value().contains("type") && - prop.value().at("type").is_string()) { - type = prop.value().at("type").get<std::string>(); - } - - auto value = type == "string" ? p.tool_arg_string_value(p.until(ARG_END)) : - p.tool_arg_value(p.until(ARG_END)); - - // skip the trailing type="..." attribute: anything up to <|sep|> - arg_choices |= p.rule("kimi-k3-arg-" + name + "-" + key, - p.tool_arg(p.tool_arg_open(p.literal(ARG_START)) + - p.tool_arg_name(p.literal(key)) + p.literal("\"") + - p.until(SEP) + p.literal(SEP) + value + - p.tool_arg_close(p.literal(ARG_END)))); - } - args = p.zero_or_more(arg_choices); - } - - // skip the trailing index="N" attribute the same way - auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) + p.literal("\"") + - p.until(SEP) + p.literal(SEP)) + - p.tool_args(args) + p.tool_close(p.literal(CALL_END))); - - tool_choices |= p.rule("kimi-k3-tool-" + name, call); - }); - - // all calls go inside one tools section, then the message is closed. the - // message closer is part of the trigger rule, or else the lazy grammar - // rejects it once tool calls have started - auto tools_section = - p.trigger_rule("kimi-k3-tool-call", p.literal(TOOLS_START) + p.one_or_more(tool_choices) + - p.literal(TOOLS_END) + p.optional(p.literal(MSG_END)) + - p.optional(p.literal(EOM_TOKEN))); - - auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section : - p.optional(tools_section); - - return start + reasoning + response + tools + trailer + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - if (function.contains("parameters")) { - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - } - }); - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOLS_START }, - }; - } - - return data; -} - -// Cohere2 MoE (a.k.a. "North Code") parser. -// -// The assistant turn is fully marker-wrapped: -// <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> -// <|START_THINKING|>{reasoning}<|END_THINKING|> -// then EITHER content: <|START_TEXT|>{content}<|END_TEXT|> -// OR tool calls: <|START_ACTION|>[ -// {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... -// ]<|END_ACTION|> -// <|END_OF_TURN_TOKEN|> -// -// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is -// the template default), so the model's output continues from *inside* the thinking block. The -// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix -// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself, -// regardless of whether they came from the generation prompt or the generated text. -static common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - const std::string TURN_START = "<|START_OF_TURN_TOKEN|>"; - const std::string TURN_END = "<|END_OF_TURN_TOKEN|>"; - const std::string CHATBOT = "<|CHATBOT_TOKEN|>"; - const std::string USER = "<|USER_TOKEN|>"; - const std::string SYSTEM = "<|SYSTEM_TOKEN|>"; - const std::string THINK_START = "<|START_THINKING|>"; - const std::string THINK_END = "<|END_THINKING|>"; - const std::string TEXT_START = "<|START_TEXT|>"; - const std::string TEXT_END = "<|END_TEXT|>"; - const std::string ACTION_START = "<|START_ACTION|>"; - const std::string ACTION_END = "<|END_ACTION|>"; - const std::string RESULT_START = "<|START_TOOL_RESULT|>"; - const std::string RESULT_END = "<|END_TOOL_RESULT|>"; - - // Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker. - const std::string GEN_PREFIX = TURN_START + CHATBOT; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.thinking_start_tag = THINK_START; - data.thinking_end_tags = {THINK_END}; - data.preserved_tokens = { - TURN_START, TURN_END, CHATBOT, USER, SYSTEM, - THINK_START, THINK_END, - TEXT_START, TEXT_END, - ACTION_START, ACTION_END, - RESULT_START, RESULT_END, - }; - - // Declare per-role message delimiters. Tool results are rendered with the - // system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before - // the plain "system" one (it is a strict superset, and the role split tries delimiters in order). - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX }, - { COMMON_CHAT_ROLE_USER, TURN_START + USER }, - { COMMON_CHAT_ROLE_TOOL, TURN_START + SYSTEM + RESULT_START }, - { COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + TEXT_START + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal(GEN_PREFIX); - auto end = p.end(); - - // The thinking block is always present (the generation prompt forces <|START_THINKING|>). - // When extracting reasoning, capture its body; otherwise keep the whole block (markers - // included) inline as content, matching reasoning_format=NONE conventions. - common_peg_parser reasoning = p.eps(); - if (extract_reasoning) { - reasoning = p.optional(p.literal(THINK_START) + - p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) + - p.optional(p.literal(THINK_END))); - } else { - reasoning = p.optional(p.content(p.literal(THINK_START) + - p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) + - p.optional(p.literal(THINK_END)))); - } - - auto text_content = has_response_format - ? p.literal(TEXT_START) + - p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + - p.optional(p.literal(TEXT_END)) - : p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END)); - - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end; - } - - auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; - - // <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|> - auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls, - /* force_tool_calls = */ true, - /* name_key = */ "tool_name", - /* args_key = */ "parameters", - /* array_wrapped = */ true, - /* function_is_key = */ false, - /* call_id_key = */ "", - /* gen_call_id_key = */ "tool_call_id", - /* parameters_order = */ { "tool_call_id", "tool_name", "parameters" }); - - // Content and tool calls are mutually exclusive in this format. - common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content }); - - return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START } - }; - } - - return data; -} - -static common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_MINIMAX_M3; - data.supports_thinking = true; - data.thinking_start_tag = "<mm:think>"; - data.thinking_end_tags = {"</mm:think>"}; - - // M3 prefixes every tool tag with the namespace token "]<]minimax[>["; - // params use the parameter name as the tag (<file_path>...</file_path>). - const std::string NS = "]<]minimax[>["; - const std::string THINK_START = "<mm:think>"; - const std::string THINK_END = "</mm:think>"; - const std::string FC_START = NS + "<tool_call>"; - const std::string FC_END = NS + "</tool_call>"; - const std::string INVOKE_END = NS + "</invoke>"; - - data.preserved_tokens = { - NS, - "<tool_call>", - "</tool_call>", - THINK_START, - THINK_END, - }; - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai" }, - { COMMON_CHAT_ROLE_USER, "]~b]user" }, - { COMMON_CHAT_ROLE_TOOL, "]~b]tool" }, - { COMMON_CHAT_ROLE_SYSTEM, "]~b]developer" }, - { COMMON_CHAT_ROLE_SYSTEM, "]~b]system" }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - const std::string GEN_PROMPT = data.generation_prompt; - - using mm3 = common_chat_peg_minimax_m3_mapper; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START); - auto end = p.end(); - - auto reasoning = p.eps(); - if (extract_reasoning) { - auto block = inputs.enable_thinking - ? p.literal(THINK_START) + p.space() + - p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END) - : p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END); - - // A turn without reasoning is prefixed with a bare </mm:think>, written either by the - // generation prompt (thinking_mode = "disabled") or by the model itself. - reasoning = p.optional(p.choice({ block, p.literal(THINK_END) })); - } - - if (has_response_format) { - auto response_format = p.rule("response-format", - p.literal("```json") + p.space() + - p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + - p.space() + p.literal("```")); - return generation_prompt + reasoning + response_format + end; - } - - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - return generation_prompt + reasoning + p.content(p.rest()) + end; - } - - auto alternatives_of = [](const json & schema) -> std::optional<json> { - for (const auto * keyword : { "oneOf", "anyOf" }) { - if (schema.contains(keyword) && schema.at(keyword).is_array() && !schema.at(keyword).empty()) { - return schema.at(keyword); - } - } - return std::nullopt; - }; - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - // The template expands argument values recursively in XML (see the to_xml() macro) - std::function<common_peg_parser(const json &, const std::string &, const std::string &)> value_of; - std::function<common_peg_parser(const json &, const std::string &)> members_of; - - auto element_of = [&](const std::string & tag, const json & schema, const std::string & rule_name) { - const std::string close = NS + "</" + tag + ">"; - return p.rule(rule_name, - p.tool_arg( - p.tool_arg_open( - p.literal(NS + "<") + - p.tool_arg_name(p.literal(tag)) + - p.literal(">")) + - value_of(schema, rule_name, close))); - }; - - value_of = [&](const json & schema, - const std::string & rule_name, - const std::string & close) -> common_peg_parser { - auto close_tag = p.tool_arg_close(p.literal(close)); - - // A string accepts anything, so a union with a string alternative is a string - if (schema_info.resolves_to_string(schema)) { - return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close); - } - - if (auto alternatives = alternatives_of(schema)) { - std::vector<common_peg_parser> choices; - - size_t index = 0; - for (const auto & alternative : *alternatives) { - const std::string alt_name = rule_name + "-" + std::to_string(index++); - - // There is a risk that this breaks streaming deltas, but that's a risk we - // assume to provide tool arg streaming. - choices.push_back(value_of(alternative, alt_name, close)); - } - - return p.choice(choices); - } - - const std::string type = schema.contains("type") && schema.at("type").is_string() - ? schema.at("type").get<std::string>() - : ""; - - if (type == "object" && schema.contains("properties")) { - return p.tag(mm3::TOOL_ARG_OBJECT, members_of(schema, rule_name)) + p.space() + close_tag; - } - - if (type == "array" && schema.contains("items")) { - const std::string item_close = NS + "</item>"; - auto item = p.rule(rule_name + "-item", - p.tag(mm3::TOOL_ARG_ITEM, - p.literal(NS + "<item>") + - value_of(schema.at("items"), rule_name + "-item", item_close))); - return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag; - } - - return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", schema, false)) + close_tag; - }; - - // Required properties in schema order, then any number of optional ones in any order. - members_of = [&](const json & schema, const std::string & rule_prefix) -> common_peg_parser { - const auto & props = schema.at("properties"); - - std::set<std::string> required; - if (schema.contains("required")) { - required = schema.at("required").get<std::set<std::string>>(); - } - - std::vector<common_peg_parser> required_elements; - std::vector<common_peg_parser> optional_elements; - for (const auto & [key, key_schema] : props.items()) { - auto element = element_of(key, key_schema, rule_prefix + "-" + key); - if (required.find(key) != required.end()) { - required_elements.push_back(element); - } else { - optional_elements.push_back(element); - } - } - - common_peg_parser members = p.eps(); - for (size_t i = 0; i < required_elements.size(); i++) { - if (i > 0) { - members = members + p.space(); - } - members = members + required_elements[i]; - } - - if (!optional_elements.empty()) { - common_peg_parser any_optional = p.choice(); - for (const auto & element : optional_elements) { - any_optional |= element; - } - members = members + p.repeat(p.space() + any_optional, 0, -1); - } - - return members; - }; - - common_peg_parser invoke_body = - params.contains("properties") ? members_of(params, "tool-" + name + "-arg") : p.eps(); - - auto func_parser = p.tool( - p.tool_open(p.literal(NS + "<invoke name=\"") + - p.tool_name(p.literal(name)) + p.literal("\">")) + - p.space() + invoke_body + p.space() + - p.tool_close(p.literal(INVOKE_END))); - - tool_choice |= p.rule("tool-" + name, func_parser); - }); - - auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; - - common_peg_parser tool_calls = p.eps(); - if (inputs.parallel_tool_calls) { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + - p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); - } else { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); - } - - if (!require_tools) { - tool_calls = p.optional(tool_calls); - } - - auto content_before_tools = p.content(p.until(FC_START)); - return generation_prompt + reasoning + content_before_tools + tool_calls + end; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START }, - }; - } - - return data; -} - -namespace workaround { - -static void map_developer_role_to_system(json & messages) { - for (auto & message : messages) { - if (message.contains("role")) { - if (message["role"] == "developer") { - message["role"] = "system"; - } - } - } -} - - -// if first message is system and template does not support it, merge it with next message -static void system_message_not_supported(json & messages) { - if (!messages.empty() && messages.front().at("role") == "system") { - if (messages.size() > 1) { - LOG_DBG("Merging system prompt into next message\n"); - auto & first_msg = messages.front(); - auto & second_msg = messages[1]; - second_msg["content"] = first_msg.at("content").get<std::string>() - + "\n" + second_msg.at("content").get<std::string>(); - messages.erase(0); - } else { - LOG_WRN("Removing system prompt due to template not supporting system role\n"); - messages.erase(0); - } - } -} - -static void requires_non_null_content(json & messages) { - GGML_ASSERT(messages.is_array()); - for (auto & message : messages) { - if (message.contains("tool_calls") && !message.contains("content")) { - message["content"] = ""; - } - } -} - -// Gemma4 uses a custom tool_responses field instead of role:tool messages. -// -// This will transform a sequence of messages: -// assistant(tool_call+) -> tool+ -> assistant(content) -// -// Into a single assistant message containing a tool_responses field: -// assistant(content + tool_call + tool_responses) -// -// This is necessary for the Gemma4 chat template to properly format the prompt. -// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4 -struct gemma4_model_turn_builder { - json & messages; - size_t pos; - json tool_calls = json::array(); - json tool_responses = json::array(); - json content; - json reasoning_content; - - gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {} - - void collect() { - // Collect the first assistant message - auto & msg = messages[pos]; - if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { - // According to the prompt formatting guide, we need to preserve reasoning_content - // between function calls. The current chat templates do not support this, but we will do it anyway. - reasoning_content = msg.at("reasoning_content"); - } - for (auto & tc : msg.at("tool_calls")) { - tool_calls.push_back(tc); - } - pos++; - - // Collect tool call results - while (pos < messages.size() && messages[pos].value("role", "") == "tool") { - collect_result(messages[pos]); - pos++; - } - - // Check if the next assistant message is the final message - if (pos < messages.size() && messages[pos].value("role", "") == "assistant") { - auto & next = messages[pos]; - if (!has_tool_calls(next) && has_content(next)) { - content = next.at("content"); - pos++; - } - } - } - - void collect_result(const json & curr) { - json response; - if (curr.contains("content")) { - const auto & content = curr.at("content"); - if (content.is_string()) { - // Try to parse the content as JSON; fall back to raw string - try { - response = json::parse(content.get<std::string>()); - } catch (...) { - response = content; - } - } else { - response = content; - } - } - - std::string name; - - // Match name with corresponding tool call - size_t idx = tool_responses.size(); - if (idx < tool_calls.size()) { - auto & tc = tool_calls[idx]; - if (tc.contains("function")) { - name = tc.at("function").value("name", ""); - } - } - - // Fallback to the tool call id - if (name.empty()) { - name = curr.value("tool_call_id", ""); - } - - tool_responses.push_back({{"name", name}, {"response", response}}); - } - - json build() { - collect(); - - json msg = { - {"role", "assistant"}, - {"tool_calls", tool_calls}, - }; - if (!tool_responses.empty()) { - msg["tool_responses"] = tool_responses; - } - if (!content.is_null()) { - msg["content"] = content; - } - if (!reasoning_content.is_null()) { - msg["reasoning_content"] = reasoning_content; - } - return msg; - } - - static bool has_content(const json & msg) { - if (!msg.contains("content") || msg.at("content").is_null()) { - return false; - } - const auto & content = msg.at("content"); - if (content.is_string() && !content.get<std::string>().empty()) { - return true; - } - if (content.is_array() && !content.empty()) { - return true; - } - return false; - } - - static bool has_tool_calls(const json & msg) { - return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty(); - } -}; - -static void convert_tool_responses_gemma4(json & messages) { - json result = json::array(); - size_t i = 0; - - while (i < messages.size()) { - auto & msg = messages[i]; - - if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") || - !msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) { - result.push_back(msg); - i++; - continue; - } - - gemma4_model_turn_builder builder(messages, i); - result.push_back(builder.build()); - i = builder.pos; - } - - messages = result; -} - -static void func_args_not_string(json & messages) { - GGML_ASSERT(messages.is_array()); - for (auto & message : messages) { - if (message.contains("tool_calls")) { - for (auto & tool_call : message["tool_calls"]) { - if (tool_call.contains("function") && tool_call["function"].contains("arguments")) { - auto & args = tool_call["function"]["arguments"]; - if (args.is_string()) { - try { - args = json::parse(args.get<std::string>()); - } catch (const std::exception & e) { - throw std::runtime_error("Failed to parse tool call arguments as JSON: " + std::string(e.what())); - } - } - } - } - } - } -} - -// Trim leading/trailing whitespace from message contents before rendering. This -// has to run on the messages (not on the rendered JSON) because templates with -// string-only content caps concatenate typed content parts into a single string -// during rendering, after which the per-part whitespace can no longer be reached. -// Both the plain string content and the text of typed content parts are trimmed. -static void trim_all_content(std::vector<common_chat_msg> & messages) { - for (auto & message : messages) { - message.content = trim_whitespace(message.content); - message.reasoning_content = trim_whitespace(message.reasoning_content); - for (auto & part : message.content_parts) { - if (part.type == "text") { - part.text = trim_whitespace(part.text); - } - } - } -} - -} - -// MiniCPM5 format: -// - Reasoning: <think>{reasoning}</think> (optional) -// - Tool calls: <function name="foo"><param name="bar">value</param></function> -static common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.preserved_tokens = { - "<function", - "<param", - "</function>", - "</param>", - "<think>", - "</think>", - }; - - data.thinking_start_tag = "<think>"; - data.thinking_end_tags = {"</think>"}; - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, - { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = "<|im_start|>assistant\n<think>\n" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "\n</think>\n\n" + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal("<|im_start|>assistant\n"); - - auto reasoning = p.eps(); - if (extract_reasoning) { - reasoning = ("<think>" << p.reasoning(p.until("</think>")) << "</think>") + p.space(); - } - - // Response format parser - if (has_response_format) { - return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema)); - } - - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - // CDATA lets a value carry characters that would otherwise close the tag (e.g. - // </param>); capture the inner text only, excluding the CDATA markers. - auto string_value = p.choice({ - p.literal("<![CDATA[") + p.ac(p.tool_arg_string_value(p.until("]]>")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("</param>")), - p.negate(p.literal("<![CDATA[")) + p.ac(p.tool_arg_string_value(p.until("</param>")) + p.tool_arg_close(p.literal("</param>")), "</param>") - }); - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - const std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto args = p.eps(); - if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - auto arg_choice = p.choice(); - for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { - auto value_parser = p.eps(); - if (schema_info.resolves_to_string(prop_schema)) { - value_parser = string_value; - } else { - value_parser = p.tool_arg_json_value( - p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false) - ) + p.tool_arg_close(p.literal("</param>")); - } - - auto arg_rule = p.tool_arg( - p.tool_arg_open(p.literal("<param name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) + - value_parser - ); - - arg_choice |= arg_rule; - } - args = p.zero_or_more(arg_choice + p.space()); - } - - auto tool_parser = p.tool( - p.tool_open(p.literal("<function name=\"") + p.tool_name(p.literal(name)) + p.literal("\">")) - << p.tool_args(args) - << p.tool_close(p.literal("</function>"))); - - tool_choice |= p.rule("tool-" + name, tool_parser); - }); - - auto max_calls = inputs.parallel_tool_calls ? -1 : 1; - auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls)); - - auto content = p.content(p.until("<function")); - - return generation_prompt + reasoning + content + tool_calls + p.end(); - } - - return generation_prompt + reasoning + p.content(p.rest()) + p.end(); - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<function" }, - }; - } - - return data; -} - -// An assistant turn is rendered as one or more messages, each -// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is -// <|eom|> (more messages follow) or <|eot|> (end of turn): -// - chain-of-thought: to=self, terminated by <|eom|> -// - final answer: to=user, terminated by <|eot|> -// The generation prompt is just "<|start|>assistant"; the model emits its own -// " to=...<|message|>". -static common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = "<|start|>assistant"; - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - - data.preserved_tokens = { - "<|start|>", "<|message|>", "<|eom|>", "<|eot|>", - // ATEM tool-call markup emitted on " to=<tool>" turns. - "<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>", - "</atem:invoke>", "</atem:function_calls>", - }; - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, - { COMMON_CHAT_ROLE_USER, "<|start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, - { COMMON_CHAT_ROLE_TOOL, "<|start|>tool" }, - }; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - // Constrained grammar whenever tools are offered. - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto start = p.rule("start", p.literal("<|start|>assistant")); - - if (!extract_reasoning && !include_grammar) { - return start + p.content(p.rest()); - } - - if (extract_reasoning) { - p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>")); - } else { - p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>")); - } - auto analysis = p.ref("analysis"); - - auto recipient = p.optional(p.literal(" to=user")); - auto final_msg = p.rule("final", recipient + p.literal("<|message|>") + - p.content(p.until_one_of({ "<|eot|>", "<|eom|>" }))); - - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto string_value = p.ac( - p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")), - "</atem:parameter>"); - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - const std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto args = p.eps(); - if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - auto arg_choice = p.choice(); - for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { - auto value_parser = p.eps(); - if (schema_info.resolves_to_string(prop_schema)) { - value_parser = string_value; - } else { - value_parser = p.tool_arg_json_value( - p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)) - + p.tool_arg_close(p.literal("</atem:parameter>")); - } - - auto arg_rule = p.tool_arg( - p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) + - value_parser); - - arg_choice |= arg_rule; - } - args = p.zero_or_more(arg_choice + p.space()); - } - - auto tool_parser = p.tool( - p.tool_open(p.literal(" to=") + p.until("<|message|>") + - p.literal("<|message|><atem:function_calls>") + p.space() + - p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space()) - << p.tool_args(args) - << p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>"))); - - tool_choice |= p.rule("tool-" + name, tool_parser); - }); - - auto tool_calls = inputs.parallel_tool_calls - ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice)) - : p.trigger_rule("tool-call", tool_choice); - - - if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { - return p.zero_or_more(start + analysis) + start + tool_calls; - } - auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls); - return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls)); - } - - return p.zero_or_more(start + analysis) + start + final_msg; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - parser.build_grammar(builder, data.grammar_lazy); - }); - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, - "<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" }, - }; - } - - return data; -} - -static json common_chat_extra_context() { - json ctx = json::object(); - std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); - std::string datetime_str = format_time(now, "%b %d %Y"); - std::string date_str = format_time(now, "%d %b %Y"); - ctx["datetime"] = datetime_str; - ctx["date_string"] = date_str; - return ctx; -} - -std::optional<common_chat_params> common_chat_try_specialized_template( - const common_chat_template & tmpl, - const std::string & src, - autoparser::generation_params & params) { - // Ministral/Mistral Large 3 - uses special reasoning structure fixes, can't use autoparser - // Note: Mistral Small 3.2 uses [CALL_ID] which Ministral doesn't have, so we can distinguish them - if (src.find("[SYSTEM_PROMPT]") != std::string::npos && src.find("[TOOL_CALLS]") != std::string::npos && - src.find("[ARGS]") != std::string::npos && src.find("[CALL_ID]") == std::string::npos) { - LOG_DBG("Using specialized template: Ministral/Magistral Large 3\n"); - return common_chat_params_init_ministral_3(tmpl, params); - } - - // GPT-OSS - has unique channel-based structure that needs dedicated handler - if (src.find("<|channel|>") != std::string::npos) { - LOG_DBG("Using specialized template: GPT-OSS\n"); - return common_chat_params_init_gpt_oss(tmpl, params); - } - - // Muse Glimmer format using " to=<recipient>" recipients and <|eom|>/<|eot|> message terminators. - if (src.find("<atem:function_calls>") != std::string::npos && src.find("<|eom|>") != std::string::npos) { - LOG_DBG("Using specialized template: Muse Glimmer\n"); - return common_chat_params_init_muse_glimmer(tmpl, params); - } - - // Functionary v3.2 - uses recipient-based format with >>>recipient\n{content} - // Detection: template has ">>>all" for content and ">>>" prefix for tool calls - if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) { - LOG_DBG("Using specialized template: Functionary v3.2\n"); - return common_chat_params_init_functionary_v3_2(tmpl, params); - } - - // Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index> - // Detection: template has "<|tool_calls_section_begin|>" and "functions." prefix in tool call IDs - if (src.find("<|tool_calls_section_begin|>") != std::string::npos && - src.find("<|tool_call_begin|>") != std::string::npos) { - LOG_DBG("Using specialized template: Kimi K2 Thinking\n"); - return common_chat_params_init_kimi_k2(tmpl, params); - } - - // Kimi K3 - the <|open|>/<|close|>/<|end_of_msg|> markers are unique to it - if (src.find("<|open|>") != std::string::npos && src.find("<|close|>") != std::string::npos && - src.find("<|end_of_msg|>") != std::string::npos) { - LOG_DBG("Using specialized template: Kimi K3\n"); - return common_chat_params_init_kimi_k3(tmpl, params); + // Ling 3.0 / Bailing V3 - <role>X</role> sections with <arg_key>/<arg_value> tagged + // tool calls. <role> sections are unique to this family among the tagged-arg templates. + if (src.find("<role>ASSISTANT</role>") != std::string::npos && + src.find("<arg_key>") != std::string::npos) { + LOG_DBG("Using specialized template: Ling 3.0 (Bailing V3)\n"); + return common_chat_params_init_ling3(tmpl, params); } // Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and @@ -3613,7 +1212,9 @@ std::optional<common_chat_params> common_chat_try_specialized_template( // Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash if (src.find("<tool_call>") != std::string::npos && src.find("<function=") != std::string::npos && - src.find("<parameter=") != std::string::npos) { + src.find("<parameter=") != std::string::npos && + // Exclude models that don't use \n between tags + src.find("'<tool_call><function=' ~ tool_call.name ~ '>'") == std::string::npos) { LOG_DBG("Using specialized template: Qwen3-Coder\n"); return common_chat_params_init_qwen3_coder(tmpl, params); } diff --git a/common/chat.h b/common/chat.h index cb39e3458f44..0e1423a5a3b1 100644 --- a/common/chat.h +++ b/common/chat.h @@ -360,6 +360,9 @@ common_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools); +// The parameters schema of a function tool. A tool without parameters, or with an empty {}, takes zero arguments. +common_json common_chat_tool_parameters(const common_json & function); + // get template caps, useful for reporting to server /props endpoint std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates); diff --git a/common/common.cpp b/common/common.cpp index 9348998b381c..6d2571db3b5e 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1637,6 +1637,11 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) { return COMMON_CONTEXT_SEQ_RM_TYPE_NO; } + if (llama_n_rs_seq(ctx) > 0) { + COM_TRC("%s", "the context supports bounded partial sequence removal\n"); + return COMMON_CONTEXT_SEQ_RM_TYPE_RS; + } + common_context_seq_rm_type res = COMMON_CONTEXT_SEQ_RM_TYPE_PART; llama_memory_clear(mem, true); @@ -1653,12 +1658,6 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) { goto done; } - if (llama_n_rs_seq(ctx) > 0) { - COM_TRC("%s", "the context supports bounded partial sequence removal\n"); - res = COMMON_CONTEXT_SEQ_RM_TYPE_RS; - goto done; - } - // try to remove the last tokens if (!llama_memory_seq_rm(mem, 0, 1, -1)) { COM_TRC("%s", "the context does not support partial sequence removal\n"); diff --git a/common/common.h b/common/common.h index 36d865fd1ab6..c4a41a1e0d28 100644 --- a/common/common.h +++ b/common/common.h @@ -809,10 +809,10 @@ struct common_params { std::string kv_tier_instance_id; // MAD-131: per-instance ID for cold-tier subdir + lockfile (default: pid) int kv_tier_cold_budget_mb = 0; // MAD-131: cap cold-pool size to N MiB (0 = no limit beyond percent-derived) - std::string hostname = "127.0.0.1"; std::string public_path = ""; // NOLINT std::string api_prefix = ""; // NOLINT std::string chat_template = ""; // NOLINT + std::vector<std::string> hostnames = {"127.0.0.1"}; bool use_jinja = true; // NOLINT // server CORS params diff --git a/common/fit.cpp b/common/fit.cpp index 759d44134bcf..e9dc9a3f844d 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -1,5 +1,6 @@ #include "fit.h" +#include "json.h" #include "log.h" #include "../src/llama-ext.h" @@ -196,9 +197,9 @@ static void common_params_fit_impl( uint32_t hp_nct = 0; // hparams.n_ctx_train uint32_t hp_nex = 0; // hparams.n_expert - // with non-unified kv, we need to take into account n_streams - // for example, if memory can hold more than model's trained context size, we must extend the n_ctx to hold enough n_streams - const uint32_t n_streams = cparams->kv_unified ? 1 : std::max<uint32_t>(1, cparams->n_seq_max); + // size the context for all sequences, but keep minimums and alignment per KV stream + const uint32_t n_seq_max = std::max<uint32_t>(1, cparams->n_seq_max); + const uint32_t n_streams = cparams->kv_unified ? 1 : n_seq_max; const bool n_ctx_auto = cparams->n_ctx == 0; dmds_t dmds_extra; // memory of the extra model, laid out on the devices of the main model @@ -268,15 +269,15 @@ static void common_params_fit_impl( dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); // saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min: - const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_streams, UINT32_MAX); + const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_seq_max, UINT32_MAX); const uint32_t n_ctx_min_total = (uint32_t) std::min<uint64_t>(uint64_t(n_ctx_min) * n_streams, UINT32_MAX); // llama_context would use only hp_nct in total for n_ctx == 0, resolve the context before measuring anything else: if (n_ctx_auto) { cparams->n_ctx = n_ctx_max; - if (n_streams > 1) { - LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n", - __func__, n_ctx_max, n_streams); + if (n_seq_max > 1) { + LOG_TRC("%s: context size unset -> using %" PRIu32 " for %" PRIu32 " sequences:\n", + __func__, n_ctx_max, n_seq_max); dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); } } @@ -920,6 +921,9 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { std::vector<std::array<std::string, 9>> table_data; table_data.reserve(devices.size()); + + // same data as the table below, for --log-jsonl consumers + common_json rows = common_json::array(); const std::string template_header = "%s: | %s | %s %s %s %s %s %s %s |\n"; const std::string template_gpu = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n"; const std::string template_other = "%s: | %s | %s %s %s = %s + %s + %s %s |\n"; @@ -994,6 +998,19 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { std::to_string(mb.context / MiB), std::to_string(mb.compute / MiB), std::to_string(unaccounted / static_cast<int64_t>(MiB))}); + + rows.push_back({ + {"kind", "device"}, + {"name", name}, + {"description", desc}, + {"total", total / MiB}, + {"free", free / MiB}, + {"self", self / MiB}, + {"model", mb.model / MiB}, + {"context", mb.context / MiB}, + {"compute", mb.compute / MiB}, + {"unaccounted", unaccounted / static_cast<int64_t>(MiB)}, + }); } // print memory breakdown for host: @@ -1009,6 +1026,15 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { std::to_string(mb_host.context / MiB), std::to_string(mb_host.compute / MiB), ""}); // unaccounted + + rows.push_back({ + {"kind", "host"}, + {"name", "Host"}, + {"self", self / MiB}, + {"model", mb_host.model / MiB}, + {"context", mb_host.context / MiB}, + {"compute", mb_host.compute / MiB}, + }); } // print memory breakdown for all remaining buffer types: @@ -1030,6 +1056,16 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { std::to_string(mb.context / MiB), std::to_string(mb.compute / MiB), ""}); // unaccounted + + rows.push_back({ + {"kind", "buffer_type"}, + {"name", name}, + {"self", self / MiB}, + {"model", mb.model / MiB}, + {"context", mb.context / MiB}, + {"compute", mb.compute / MiB}, + }); + seen_buffer_types.insert(buft); } @@ -1047,6 +1083,11 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { __func__, td[1].c_str(), td[2].c_str(), td[3].c_str(), td[4].c_str(), td[5].c_str(), td[6].c_str(), td[7].c_str(), td[8].c_str()); } + + LOG_JSON("fit_memory_breakdown", common_json({ + {"unit", "MiB"}, + {"rows", rows}, + })); } void common_fit_print( diff --git a/common/hf-cache.cpp b/common/hf-cache.cpp index 50d6dd6105c4..12d4fcc80447 100644 --- a/common/hf-cache.cpp +++ b/common/hf-cache.cpp @@ -62,6 +62,15 @@ static fs::path get_cache_directory() { return cache; } +std::string get_cache_path() { +#if defined(__cpp_lib_char8_t) + const std::u8string u8str = get_cache_directory().u8string(); + return std::string(reinterpret_cast<const char *>(u8str.data()), u8str.size()); +#else + return get_cache_directory().u8string(); +#endif +} + static std::string folder_name_to_repo(const std::string & folder) { constexpr std::string_view prefix = "models--"; if (folder.rfind(prefix, 0)) { @@ -393,8 +402,8 @@ static std::string get_cached_ref(const fs::path & repo_path) { } hf_files get_cached_files(const std::string & repo_id) { - fs::path cache_dir = get_cache_directory(); - if (!fs::exists(cache_dir)) { + const fs::path cache_path = get_cache_directory(); + if (!fs::exists(cache_path)) { return {}; } @@ -405,7 +414,7 @@ hf_files get_cached_files(const std::string & repo_id) { hf_files files; - for (const auto & repo : fs::directory_iterator(cache_dir)) { + for (const auto & repo : fs::directory_iterator(cache_path)) { if (!repo.is_directory()) { continue; } diff --git a/common/hf-cache.h b/common/hf-cache.h index 42c9c6ce34f0..41842db788cb 100644 --- a/common/hf-cache.h +++ b/common/hf-cache.h @@ -32,4 +32,7 @@ std::string finalize_file(const hf_file & file); // Remove the entire cached directory for a repo, returns true if removed bool remove_cached_repo(const std::string & repo_id); +// Returns the HuggingFace hub cache path +std::string get_cache_path(); + } // namespace hf_cache diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index 9971c021e188..c5962ab77685 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -117,6 +117,7 @@ caps caps_get(jinja::program & prog) { JJ_DEBUG("%s\n", ">>> Running capability check: typed content"); + bool checks_for_string = false; static const std::string content_marker = "STRING_MARKER"; // case: typed content support @@ -136,6 +137,10 @@ caps caps_get(jinja::program & prog) { [&](context &, bool success, value & messages, value &, const std::string & rendered) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); + if (has_op(content, "test_is_string")) { + // checked if content is string + checks_for_string = true; + } bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access"); if (used_as_array) { // accessed as an array @@ -151,6 +156,33 @@ caps caps_get(jinja::program & prog) { } ); + if (checks_for_string) { + caps_try_execute( + prog, + [&]() { + // messages + return json::array({ + { + {"role", "user"}, + {"content", json::array({ + })} + } + }); + }, + nullptr, // ctx_fn + nullptr, // tools_fn + [&](context &, bool success, value & messages, value &, const std::string &) { + auto & content = messages->at(0)->at("content"); + caps_print_stats(content, "messages[0].content"); + bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access"); + if (used_as_array && success) { + // accessed as an array + result.supports_typed_content = true; + } + } + ); + } + JJ_DEBUG("%s\n", ">>> Running capability check: system prompt"); // case: system prompt support diff --git a/common/jinja/parser.cpp b/common/jinja/parser.cpp index 2b25654a7a0a..1ddceede9124 100644 --- a/common/jinja/parser.cpp +++ b/common/jinja/parser.cpp @@ -437,7 +437,8 @@ class parser { } statement_ptr parse_filter_expression() { - auto operand = parse_call_member_expression(); + // Filters/tests bind outside unary so -n|abs is (-n)|abs, not -(n|abs). + auto operand = parse_unary_expression(); while (is(token::pipe)) { size_t start_pos = current; ++current; // consume pipe @@ -448,6 +449,15 @@ class parser { return operand; } + statement_ptr parse_unary_expression() { + if (is(token::unary_operator)) { + size_t start_pos = current; + auto op = next(); + return mk_stmt<unary_expression>(start_pos, op, parse_unary_expression()); + } + return parse_call_member_expression(); + } + statement_ptr parse_call_member_expression() { // Handle member expressions recursively auto member = parse_member_expression(parse_primary_expression()); diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index 4ce79e32aa7c..e2cab8aa6442 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -51,7 +51,7 @@ static void ensure_key_type_allowed(const value & val) { } // execute with error handling -value statement::execute(context & ctx) { +value statement::execute(context & ctx) const { try { return execute_impl(ctx); } catch (const continue_statement::signal & /* ex */) { @@ -80,7 +80,7 @@ value statement::execute(context & ctx) { } } -value identifier::execute_impl(context & ctx) { +value identifier::execute_impl(context & ctx) const { auto it = ctx.get_val(val); auto builtins = global_builtins(); if (!it->is_undefined()) { @@ -98,7 +98,7 @@ value identifier::execute_impl(context & ctx) { } } -value object_literal::execute_impl(context & ctx) { +value object_literal::execute_impl(context & ctx) const { auto obj = mk_val<value_object>(); for (const auto & pair : val) { value key = pair.first->execute(ctx); @@ -109,7 +109,7 @@ value object_literal::execute_impl(context & ctx) { return obj; } -value binary_expression::execute_impl(context & ctx) { +value binary_expression::execute_impl(context & ctx) const { value left_val = left->execute(ctx); // Logical operators @@ -167,6 +167,12 @@ value binary_expression::execute_impl(context & ctx) { } throw std::runtime_error("Cannot perform operation " + op.value + " on undefined values"); } else if (is_val<value_none>(left_val) || is_val<value_none>(right_val)) { + if (!is_val<value_none>(right_val) && (op.value == "in" || op.value == "not in")) { + // case: none in {'low': 1} + // A null left operand is looked up like any other value. + bool member = test_is_in(); + return mk_val<value_bool>(op.value == "in" ? member : !member); + } if (op.value == "+" || op.value == "~") { value res = mk_val<value_undefined>(); if (workaround_concat_null_with_str(res)) { @@ -311,9 +317,7 @@ static value try_builtin_func(context & ctx, const std::string & name, value & i throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type()); } -value filter_expression::execute_impl(context & ctx) { - value input = operand ? operand->execute(ctx) : val; - +static value apply_filter(context & ctx, const statement_ptr & filter, value input) { JJ_DEBUG("Applying filter to %s", input->type().c_str()); auto set_filter_alias = [](auto & filter_id) { @@ -369,22 +373,21 @@ value filter_expression::execute_impl(context & ctx) { } } -value filter_statement::execute_impl(context & ctx) { +value filter_expression::execute_impl(context & ctx) const { + return apply_filter(ctx, filter, operand->execute(ctx)); +} + +value filter_statement::execute_impl(context & ctx) const { // eval body as string, then apply filter auto body_val = exec_statements(body, ctx); value_string parts = mk_val<value_string>(); gather_string_parts_recursive(body_val, parts); JJ_DEBUG("FilterStatement: applying filter to body string of length %zu", parts->val_str.length()); - filter_expression filter_expr(std::move(parts), std::move(filter)); - value out = filter_expr.execute(ctx); - - // this node can be reused later, make sure filter is preserved - this->filter = std::move(filter_expr.filter); - return out; + return apply_filter(ctx, filter, parts); } -value test_expression::execute_impl(context & ctx) { +value test_expression::execute_impl(context & ctx) const { // NOTE: "value is something" translates to function call "test_is_something(value)" const auto & builtins = global_builtins(); @@ -412,12 +415,18 @@ value test_expression::execute_impl(context & ctx) { throw std::runtime_error("Invalid test expression"); } - auto it = builtins.find("test_is_" + test_id); - JJ_DEBUG("Test expression %s '%s' %s (using function 'test_is_%s')", operand->type().c_str(), test_id.c_str(), negate ? "(negate)" : "", test_id.c_str()); + const std::string test_name = "test_is_" + test_id; + auto it = builtins.find(test_name); + JJ_DEBUG("Test expression %s '%s' %s (using function '%s')", operand->type().c_str(), test_id.c_str(), negate ? "(negate)" : "", test_name.c_str()); if (it == builtins.end()) { throw std::runtime_error("Unknown test '" + test_id + "'"); } + if (ctx.is_get_stats) { + value_t::stats_t::mark_used(input); + input->stats.ops.insert(test_name); + } + auto res = it->second(args); if (negate) { @@ -427,7 +436,7 @@ value test_expression::execute_impl(context & ctx) { } } -value unary_expression::execute_impl(context & ctx) { +value unary_expression::execute_impl(context & ctx) const { value operand_val = argument->execute(ctx); JJ_DEBUG("Executing unary expression with operator '%s'", op.value.c_str()); @@ -441,12 +450,17 @@ value unary_expression::execute_impl(context & ctx) { } else { throw std::runtime_error("Unary - operator requires numeric operand"); } + } else if (op.value == "+") { + if (is_val<value_int>(operand_val) || is_val<value_float>(operand_val)) { + return operand_val; + } + throw std::runtime_error("Unary + operator requires numeric operand"); } throw std::runtime_error("Unknown unary operator '" + op.value + "'"); } -value if_statement::execute_impl(context & ctx) { +value if_statement::execute_impl(context & ctx) const { value test_val = test->execute(ctx); auto out = mk_val<value_array>(); @@ -467,20 +481,14 @@ value if_statement::execute_impl(context & ctx) { return str; } -value for_statement::execute_impl(context & ctx) { +value for_statement::execute_impl(context & ctx) const { context scope(ctx); // new scope for loop variables - jinja::select_expression * select_expr = cast_stmt<select_expression>(iterable); + const jinja::select_expression * select_expr = cast_stmt<select_expression>(iterable); statement_ptr test_expr_nullptr; - statement_ptr & iter_expr = [&]() -> statement_ptr & { - auto tmp = cast_stmt<select_expression>(iterable); - return tmp ? tmp->lhs : iterable; - }(); - statement_ptr & test_expr = [&]() -> statement_ptr & { - auto tmp = cast_stmt<select_expression>(iterable); - return tmp ? tmp->test : test_expr_nullptr; - }(); + const statement_ptr & iter_expr = select_expr ? select_expr->lhs : iterable; + const statement_ptr & test_expr = select_expr ? select_expr->test : test_expr_nullptr; JJ_DEBUG("Executing for statement, iterable type: %s", iter_expr->type().c_str()); @@ -633,7 +641,7 @@ value for_statement::execute_impl(context & ctx) { return str; } -value set_statement::execute_impl(context & ctx) { +value set_statement::execute_impl(context & ctx) const { auto rhs = val ? val->execute(ctx) : exec_statements(body, ctx); if (is_stmt<identifier>(assignee)) { @@ -732,7 +740,7 @@ static inline void bind_parameters(const std::string & name, const statements & } } -value macro_statement::execute_impl(context & ctx) { +value macro_statement::execute_impl(context & ctx) const { if (!is_stmt<identifier>(this->name)) { throw std::runtime_error("Macro name must be an identifier"); } @@ -755,7 +763,7 @@ value macro_statement::execute_impl(context & ctx) { return mk_val<value_undefined>(); } -value call_statement::execute_impl(context & ctx) { +value call_statement::execute_impl(context & ctx) const { auto call_expr = cast_stmt<call_expression>(this->call); if (!call_expr) { throw std::runtime_error("Call statement requires a valid call expression"); @@ -795,7 +803,7 @@ value call_statement::execute_impl(context & ctx) { return callee_func->invoke(args); } -value member_expression::execute_impl(context & ctx) { +value member_expression::execute_impl(context & ctx) const { value object = this->object->execute(ctx); value property; @@ -830,6 +838,12 @@ value member_expression::execute_impl(context & ctx) { } else { property = this->property->execute(ctx); } + } else if (is_stmt<integer_literal>(this->property)) { + // syntax: obj.index + property = mk_val<value_int>(cast_stmt<integer_literal>(this->property)->val); + if (property->as_int() < 0) { + throw std::runtime_error("Static member property cannot be negative"); + } } else { // syntax: obj.prop if (!is_stmt<identifier>(this->property)) { @@ -922,7 +936,7 @@ value member_expression::execute_impl(context & ctx) { return val; } -value call_expression::execute_impl(context & ctx) { +value call_expression::execute_impl(context & ctx) const { // gather arguments func_args args(ctx); for (auto & arg_stmt : this->args) { @@ -940,7 +954,7 @@ value call_expression::execute_impl(context & ctx) { return callee_func->invoke(args); } -value keyword_argument_expression::execute_impl(context & ctx) { +value keyword_argument_expression::execute_impl(context & ctx) const { if (!is_stmt<identifier>(key)) { throw std::runtime_error("Keyword argument key must be identifiers"); } @@ -964,7 +978,7 @@ std::string runtime::debug_dump_program(const program & prog, const std::string return std::string(lvl * 2, ' '); }; - ctx.visitor = [&](bool is_leaf, statement * node, std::vector<visitor_pair> children) { + ctx.visitor = [&](bool is_leaf, const statement * node, std::vector<visitor_pair> children) { oss << indent(lvl) << node->type() << ":\n"; lvl++; if (is_leaf) { diff --git a/common/jinja/runtime.h b/common/jinja/runtime.h index 69bd683c68f8..bbd0c5caeacc 100644 --- a/common/jinja/runtime.h +++ b/common/jinja/runtime.h @@ -48,9 +48,9 @@ const T * cast_stmt(const statement_ptr & ptr) { void enable_debug(bool enable); // for visiting AST nodes -// function signature: void(bool is_leaf, statement * node, pair of <label, children>) -using visitor_pair = std::pair<std::string, std::vector<statement *>>; -using visitor_fn = std::function<void(bool, statement *, std::vector<visitor_pair>)>; +// function signature: void(bool is_leaf, const statement * node, pair of <label, children>) +using visitor_pair = std::pair<std::string, std::vector<const statement *>>; +using visitor_fn = std::function<void(bool, const statement *, std::vector<visitor_pair>)>; struct context { std::shared_ptr<std::string> src; // for debugging; use shared_ptr to avoid copying on scope creation @@ -107,8 +107,8 @@ struct context { }; // utils for visiting AST nodes -static std::vector<statement *> stmts_to_ptr(const statements & stmts) { - std::vector<statement *> children; +static std::vector<const statement *> stmts_to_ptr(const statements & stmts) { + std::vector<const statement *> children; for (const auto & stmt : stmts) { children.push_back(stmt.get()); } @@ -117,17 +117,18 @@ static std::vector<statement *> stmts_to_ptr(const statements & stmts) { /** * Base class for all nodes in the AST. + * The AST is shared between threads, so visit and execute must be const. */ struct statement { size_t pos; // position in source, for debugging virtual ~statement() = default; virtual std::string type() const { return "Statement"; } - virtual void visit(context & ctx) { ctx.visitor(true, this, {}); } + virtual void visit(context & ctx) const { ctx.visitor(true, this, {}); } // execute_impl must be overridden by derived classes - virtual value execute_impl(context &) { throw_exec_error(); } + virtual value execute_impl(context &) const { throw_exec_error(); } // execute is the public method to execute a statement with error handling - value execute(context &); + value execute(context &) const; private: [[noreturn]] void throw_exec_error() const { @@ -166,7 +167,7 @@ struct program : public statement { program() = default; explicit program(statements && body) : body(std::move(body)) {} std::string type() const override { return "Program"; } - [[noreturn]] value execute_impl(context &) override { + [[noreturn]] value execute_impl(context &) const override { throw std::runtime_error("Cannot execute program directly, use jinja::runtime instead"); } }; @@ -182,8 +183,8 @@ struct if_statement : public statement { } std::string type() const override { return "If"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"test", {test.get()}}, {"body", stmts_to_ptr(body)}, @@ -213,8 +214,8 @@ struct for_statement : public statement { } std::string type() const override { return "For"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"loopvar", {loopvar.get()}}, {"iterable", {iterable.get()}}, @@ -233,7 +234,7 @@ struct break_statement : public statement { } }; - [[noreturn]] value execute_impl(context &) override { + [[noreturn]] value execute_impl(context &) const override { throw break_statement::signal(); } }; @@ -247,7 +248,7 @@ struct continue_statement : public statement { } }; - [[noreturn]] value execute_impl(context &) override { + [[noreturn]] value execute_impl(context &) const override { throw continue_statement::signal(); } }; @@ -255,7 +256,7 @@ struct continue_statement : public statement { // do nothing struct noop_statement : public statement { std::string type() const override { return "Noop"; } - value execute_impl(context &) override { + value execute_impl(context &) const override { return mk_val<value_undefined>(); } }; @@ -272,8 +273,8 @@ struct set_statement : public statement { } std::string type() const override { return "Set"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"assignee", {assignee.get()}}, {"value", {val.get()}}, @@ -294,8 +295,8 @@ struct macro_statement : public statement { } std::string type() const override { return "Macro"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"name", {name.get()}}, {"args", stmts_to_ptr(args)}, @@ -308,7 +309,7 @@ struct comment_statement : public statement { std::string val; explicit comment_statement(const std::string & v) : val(v) {} std::string type() const override { return "Comment"; } - value execute_impl(context &) override { + value execute_impl(context &) const override { return mk_val<value_undefined>(); } }; @@ -318,7 +319,7 @@ struct comment_statement : public statement { // Represents an omitted expression in a computed member, e.g. `a[]`. struct blank_expression : public expression { std::string type() const override { return "BlankExpression"; } - value execute_impl(context &) override { + value execute_impl(context &) const override { return mk_val<value_undefined>(); } }; @@ -334,8 +335,8 @@ struct member_expression : public expression { chk_type<expression>(this->property); } std::string type() const override { return "MemberExpression"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"object", {object.get()}}, {"property", {property.get()}} @@ -353,8 +354,8 @@ struct call_expression : public expression { for (const auto& arg : this->args) chk_type<expression>(arg); } std::string type() const override { return "CallExpression"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"callee", {callee.get()}}, {"args", stmts_to_ptr(args)} @@ -369,7 +370,7 @@ struct identifier : public expression { std::string val; explicit identifier(const std::string & val) : val(val) {} std::string type() const override { return "Identifier"; } - value execute_impl(context & ctx) override; + value execute_impl(context & ctx) const override; }; // Literals @@ -378,7 +379,7 @@ struct integer_literal : public expression { int64_t val; explicit integer_literal(int64_t val) : val(val) {} std::string type() const override { return "IntegerLiteral"; } - value execute_impl(context &) override { + value execute_impl(context &) const override { return mk_val<value_int>(val); } }; @@ -387,7 +388,7 @@ struct float_literal : public expression { double val; explicit float_literal(double val) : val(val) {} std::string type() const override { return "FloatLiteral"; } - value execute_impl(context &) override { + value execute_impl(context &) const override { return mk_val<value_float>(val); } }; @@ -396,7 +397,7 @@ struct string_literal : public expression { std::string val; explicit string_literal(const std::string & val) : val(val) {} std::string type() const override { return "StringLiteral"; } - value execute_impl(context &) override { + value execute_impl(context &) const override { return mk_val<value_string>(val); } }; @@ -407,7 +408,7 @@ struct array_literal : public expression { for (const auto& item : this->val) chk_type<expression>(item); } std::string type() const override { return "ArrayLiteral"; } - value execute_impl(context & ctx) override { + value execute_impl(context & ctx) const override { auto arr = mk_val<value_array>(); for (const auto & item_stmt : val) { arr->push_back(item_stmt->execute(ctx)); @@ -422,7 +423,7 @@ struct tuple_literal : public expression { for (const auto& item : this->val) chk_type<expression>(item); } std::string type() const override { return "TupleLiteral"; } - value execute_impl(context & ctx) override { + value execute_impl(context & ctx) const override { auto arr = mk_val<value_array>(); for (const auto & item_stmt : val) { arr->push_back(item_stmt->execute(ctx)); @@ -441,7 +442,7 @@ struct object_literal : public expression { } } std::string type() const override { return "ObjectLiteral"; } - value execute_impl(context & ctx) override; + value execute_impl(context & ctx) const override; }; // Complex Expressions @@ -462,8 +463,8 @@ struct binary_expression : public expression { chk_type<expression>(this->right); } std::string type() const override { return "BinaryExpression"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"left", {left.get()}}, {"right", {right.get()}} @@ -476,10 +477,7 @@ struct binary_expression : public expression { * Operator precedence: https://github.com/pallets/jinja/issues/379#issuecomment-168076202 */ struct filter_expression : public expression { - // either an expression or a value is allowed statement_ptr operand; - value_string val; // will be set by filter_statement - statement_ptr filter; filter_expression(statement_ptr && operand, statement_ptr && filter) @@ -488,14 +486,9 @@ struct filter_expression : public expression { chk_type<identifier, call_expression>(this->filter); } - filter_expression(value_string && val, statement_ptr && filter) - : val(std::move(val)), filter(std::move(filter)) { - chk_type<identifier, call_expression>(this->filter); - } - std::string type() const override { return "FilterExpression"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"operand", {operand.get()}}, {"filter", {filter.get()}} @@ -512,8 +505,8 @@ struct filter_statement : public statement { chk_type<identifier, call_expression>(this->filter); } std::string type() const override { return "FilterStatement"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"filter", {filter.get()}}, {"body", stmts_to_ptr(body)} @@ -537,14 +530,14 @@ struct select_expression : public expression { chk_type<expression>(this->test); } std::string type() const override { return "SelectExpression"; } - value execute_impl(context & ctx) override { + value execute_impl(context & ctx) const override { auto predicate = test->execute_impl(ctx); if (!predicate->as_bool()) { return mk_val<value_undefined>(); } return lhs->execute_impl(ctx); } - void visit(context & ctx) override { + void visit(context & ctx) const override { ctx.visitor(false, this, { {"lhs", {lhs.get()}}, {"test", {test.get()}} @@ -567,8 +560,8 @@ struct test_expression : public expression { chk_type<identifier, call_expression>(this->test); } std::string type() const override { return "TestExpression"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"operand", {operand.get()}}, {"test", {test.get()}} @@ -588,8 +581,8 @@ struct unary_expression : public expression { chk_type<expression>(this->argument); } std::string type() const override { return "UnaryExpression"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"argument", {argument.get()}} }); @@ -608,10 +601,10 @@ struct slice_expression : public expression { chk_type<expression>(this->step_expr); } std::string type() const override { return "SliceExpression"; } - [[noreturn]] value execute_impl(context &) override { + [[noreturn]] value execute_impl(context &) const override { throw std::runtime_error("must be handled by MemberExpression"); } - void visit(context & ctx) override { + void visit(context & ctx) const override { ctx.visitor(false, this, { {"start_expr", {start_expr.get()}}, {"stop_expr", {stop_expr.get()}}, @@ -630,8 +623,8 @@ struct keyword_argument_expression : public expression { chk_type<expression>(this->val); } std::string type() const override { return "KeywordArgumentExpression"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"key", {key.get()}}, {"val", {val.get()}} @@ -645,7 +638,7 @@ struct spread_expression : public expression { chk_type<expression>(this->argument); } std::string type() const override { return "SpreadExpression"; } - void visit(context & ctx) override { + void visit(context & ctx) const override { ctx.visitor(false, this, { {"argument", {argument.get()}} }); @@ -663,8 +656,8 @@ struct call_statement : public statement { for (const auto & arg : this->caller_args) chk_type<expression>(arg); } std::string type() const override { return "CallStatement"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { ctx.visitor(false, this, { {"call", {call.get()}}, {"caller_args", stmts_to_ptr(caller_args)}, @@ -685,7 +678,7 @@ struct ternary_expression : public expression { chk_type<expression>(this->false_expr); } std::string type() const override { return "Ternary"; } - value execute_impl(context & ctx) override { + value execute_impl(context & ctx) const override { value cond_val = condition->execute(ctx); if (cond_val->as_bool()) { return true_expr->execute(ctx); @@ -693,7 +686,7 @@ struct ternary_expression : public expression { return false_expr->execute(ctx); } } - void visit(context & ctx) override { + void visit(context & ctx) const override { ctx.visitor(false, this, { {"condition", {condition.get()}}, {"true_expr", {true_expr.get()}}, diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index a7a18857d713..3349ae4fe0cc 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -1,5 +1,7 @@ #include "json-schema-to-grammar.h" #include "common.h" +#include "trie.h" +#include "unicode.h" #include <algorithm> #include <limits> @@ -319,7 +321,8 @@ static size_t gbnf_escape_length(const std::string & pattern, size_t pos) { case 'x': n_hex = 2; break; case 'u': n_hex = 4; break; case 'U': n_hex = 8; break; - case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']': + // keep in sync with parse_char() in src/llama-grammar.cpp + case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']': case '-': return 2; default: return 0; @@ -336,18 +339,20 @@ static size_t gbnf_escape_length(const std::string & pattern, size_t pos) { return 2 + n_hex; } -class common_schema_converter { +class common_chat_schema_converter { private: - friend class common_schema_info; friend std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options); - std::function<json(const std::string &)> _fetch_json; bool _dotall; std::map<std::string, std::string> _rules; - std::unordered_map<std::string, json> _refs; std::unordered_set<std::string> _refs_being_resolved; std::vector<std::string> _errors; std::vector<std::string> _warnings; + template <typename T> + static const T & as(const common_chat_schema & node) { + return static_cast<const T &>(node); + } + std::string _add_rule(const std::string & name, const std::string & rule) { std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-"); if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) { @@ -363,11 +368,11 @@ class common_schema_converter { return key; } - std::string _generate_union_rule(const std::string & name, const std::vector<json> & alt_schemas) { + std::string _generate_union_rule(const std::string & name, const std::vector<common_chat_schema_ptr> & alt_schemas) { std::vector<std::string> rules; rules.reserve(alt_schemas.size()); for (size_t i = 0; i < alt_schemas.size(); i++) { - rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i))); + rules.push_back(visit(*alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i))); } return string_join(rules, " | "); } @@ -634,85 +639,68 @@ class common_schema_converter { -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["] */ std::string _not_strings(const std::vector<std::string> & strings) { - - struct TrieNode { - std::map<char, TrieNode> children; - bool is_end_of_string; - - TrieNode() : is_end_of_string(false) {} - - void insert(const std::string & string) { - auto *node = this; - for (char c : string) { - node = &node->children[c]; - } - node->is_end_of_string = true; - } - }; - - TrieNode trie; - for (const auto & s : strings) { - trie.insert(s); - } + common_trie trie(strings); std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); std::ostringstream out; out << "[\"] ( "; - std::function<void(const TrieNode &)> visit = [&](const TrieNode & node) { - std::ostringstream rejects; + std::function<void(size_t)> visit = [&](size_t idx) { + const auto & node = trie.nodes[idx]; + std::string rejects; auto first = true; - for (const auto & kv : node.children) { - rejects << kv.first; + for (const auto & [cpt, child] : node.children) { + std::string c = common_unicode_cpt_to_utf8(cpt); + rejects += c; if (first) { first = false; } else { out << " | "; } - out << "[" << kv.first << "]"; - if (!kv.second.children.empty()) { + out << "[" << c << "]"; + if (!trie.nodes[child].children.empty()) { out << " ("; - visit(kv.second); + visit(child); out << ")"; - } else if (kv.second.is_end_of_string) { + } else { out << " " << char_rule << "+"; } } if (!node.children.empty()) { - if (!first) { - out << " | "; - } - out << "[^\"" << rejects.str() << "] " << char_rule << "*"; + out << " | [^\"" << rejects << "] " << char_rule << "*"; } }; - visit(trie); + visit(0); out << " )"; - if (!trie.is_end_of_string) { + if (trie.nodes[0].pattern < 0) { out << "?"; } out << " [\"]"; return out.str(); } - std::string _resolve_ref(const std::string & ref) { - auto it = ref.find('#'); - std::string ref_fragment = it != std::string::npos ? ref.substr(it + 1) : ref; + std::string _resolve_ref(const common_chat_schema_ref & schema) { + auto it = schema.ref.find('#'); + std::string ref_fragment = it != std::string::npos ? schema.ref.substr(it + 1) : schema.ref; static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)"); std::string ref_name = "ref" + std::regex_replace(ref_fragment, nonalphanumeric_regex, "-"); - if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) { - _refs_being_resolved.insert(ref); - json resolved = _refs[ref]; - ref_name = visit(resolved, ref_name); - _refs_being_resolved.erase(ref); + if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(schema.ref) == _refs_being_resolved.end()) { + if (!schema.target) { + _errors.push_back("Unresolved $ref " + schema.ref); + return ""; + } + _refs_being_resolved.insert(schema.ref); + ref_name = visit(*schema.target, ref_name); + _refs_being_resolved.erase(schema.ref); } return ref_name; } std::string _build_object_rule( - const std::vector<std::pair<std::string, json>> & properties, + const std::vector<std::pair<std::string, const common_chat_schema *>> & properties, const std::unordered_set<std::string> & required, const std::string & name, - const json & additional_properties) + const common_chat_schema * additional_properties) { std::vector<std::string> required_props; std::vector<std::string> optional_props; @@ -722,7 +710,7 @@ class common_schema_converter { const auto &prop_name = kv.first; const auto &prop_schema = kv.second; - std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name); + std::string prop_rule_name = visit(*prop_schema, name + (name.empty() ? "" : "-") + prop_name); prop_kv_rule_names[prop_name] = _add_rule( name + (name.empty() ? "" : "-") + prop_name + "-kv", format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name @@ -734,10 +722,10 @@ class common_schema_converter { } prop_names.push_back(prop_name); } - if ((additional_properties.is_boolean() && additional_properties.get<bool>()) || additional_properties.is_object()) { + if (additional_properties) { std::string sub_name = name + (name.empty() ? "" : "-") + "additional"; std::string value_rule = - additional_properties.is_object() ? visit(additional_properties, sub_name + "-value") + additional_properties->kind() != common_chat_schema::KIND_ANY ? visit(*additional_properties, sub_name + "-value") : _add_primitive("value", PRIMITIVE_RULES.at("value")); auto key_rule = @@ -825,267 +813,163 @@ class common_schema_converter { } public: - common_schema_converter( - const std::function<json(const std::string &)> & fetch_json, - bool dotall) - : _fetch_json(fetch_json), _dotall(dotall) - { + explicit common_chat_schema_converter(bool dotall) : _dotall(dotall) { _rules["space"] = SPACE_RULE; } - void resolve_refs(json & schema, const std::string & url) { - /* - * Resolves all $ref fields in the given schema, fetching any remote schemas, - * replacing each $ref with absolute reference URL and populates _refs with the - * respective referenced (sub)schema dictionaries. - */ - std::function<void(json &)> visit_refs = [&](json & n) { - if (n.is_array()) { - for (auto & x : n) { - visit_refs(x); - } - } else if (n.is_object()) { - if (n.contains("$ref")) { - std::string ref = n["$ref"]; - if (_refs.find(ref) == _refs.end()) { - json target; - if (ref.find("https://") == 0) { - std::string base_url = ref.substr(0, ref.find('#')); - auto it = _refs.find(base_url); - if (it != _refs.end()) { - target = it->second; - } else { - // Fetch the referenced schema and resolve its refs - auto referenced = _fetch_json(ref); - resolve_refs(referenced, base_url); - _refs[base_url] = referenced; - } - if (ref.find('#') == std::string::npos || ref.substr(ref.find('#') + 1).empty()) { - return; - } - } else if (ref.find("#/") == 0) { - target = schema; - n["$ref"] = url + ref; - ref = url + ref; - } else { - _errors.push_back("Unsupported ref: " + ref); - return; - } - std::string pointer = ref.substr(ref.find('#') + 1); - std::vector<std::string> tokens = string_split(pointer, "/"); - for (size_t i = 1; i < tokens.size(); ++i) { - const std::string& sel = tokens[i]; - if (target.is_object() && target.contains(sel)) { - target = target[sel]; - } else if (target.is_array()) { - size_t sel_index; - try { - sel_index = std::stoull(sel); - } catch (const std::invalid_argument & e) { - sel_index = target.size(); - } - if (sel_index >= target.size()) { - _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump()); - return; - } - target = target[sel_index]; - } else { - _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump()); - return; - } - } - _refs[ref] = target; - } - } else { - for (const auto & kv : n.items()) { - visit_refs(kv.value()); - } - } - } - }; - - visit_refs(schema); + std::string add_schema(const std::string & name, const common_chat_schema & schema) { + return visit(schema, name); } static std::string _generate_constant_rule(const json & value) { return format_literal(value.dump()); } - std::string visit(const json & schema, const std::string & name) { - json schema_type = schema.contains("type") ? schema["type"] : json(); - std::string schema_format = schema.contains("format") ? schema["format"].get<std::string>() : ""; - std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name; + std::string _visit_primitive(const std::string & rule_name, const std::string & type) { + return _add_primitive(rule_name == "root" ? "root" : type, PRIMITIVE_RULES.at(type)); + } - if (schema.contains("$ref")) { - return _add_rule(rule_name, _resolve_ref(schema["$ref"])); - } - if (schema.contains("oneOf") || schema.contains("anyOf")) { - const json & alts = schema.contains("oneOf") ? schema.at("oneOf") : schema.at("anyOf"); - std::vector<json> alt_schemas; - for (const auto & alt : alts) { - alt_schemas.push_back(alt); - } - return _add_rule(rule_name, _generate_union_rule(name, alt_schemas)); - } - if (schema_type.is_array()) { - std::vector<json> schema_types; - for (const auto & t : schema_type) { - json schema_copy(schema); - schema_copy["type"] = t; - schema_types.push_back(schema_copy); - } - return _add_rule(rule_name, _generate_union_rule(name, schema_types)); - } - if (schema.contains("const")) { - return _add_rule(rule_name, _generate_constant_rule(schema["const"])); - } - if (schema.contains("enum")) { - std::vector<std::string> enum_values; - for (const auto & v : schema["enum"]) { - enum_values.push_back(_generate_constant_rule(v)); - } - return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")"); - } - if ((schema_type.is_null() || schema_type == "object") - && (schema.contains("properties") || - (schema.contains("additionalProperties") && schema["additionalProperties"] != true))) { - std::unordered_set<std::string> required; - if (schema.contains("required") && schema["required"].is_array()) { - for (const auto & item : schema["required"]) { - if (item.is_string()) { - required.insert(item.get<std::string>()); + std::string _visit_all_of(const common_chat_schema_all_of & schema, const std::string & name, const std::string & rule_name) { + std::unordered_set<std::string> required; + std::vector<std::pair<std::string, const common_chat_schema *>> properties; + std::map<std::string, size_t> enum_values; + std::function<void(const common_chat_schema &, bool)> add_component = [&](const common_chat_schema & comp, bool is_required) { + if (comp.kind() == common_chat_schema::KIND_REF) { + if (const auto * target = as<common_chat_schema_ref>(comp).target) { + add_component(*target, is_required); + } + } else if (comp.kind() == common_chat_schema::KIND_OBJECT) { + for (const auto & prop : as<common_chat_schema_object>(comp).properties) { + properties.emplace_back(prop.name, prop.schema.get()); + if (is_required) { + required.insert(prop.name); } } + } else if (comp.kind() == common_chat_schema::KIND_ENUM) { + for (const auto & v : as<common_chat_schema_enum>(comp).values) { + enum_values[_generate_constant_rule(v)] += 1; + } } - std::vector<std::pair<std::string, json>> properties; - if (schema.contains("properties")) { - for (const auto & prop : schema["properties"].items()) { - properties.emplace_back(prop.key(), prop.value()); + }; + for (const auto & child : schema.children) { + if (child->kind() == common_chat_schema::KIND_ANY_OF) { + for (const auto & alt : as<common_chat_schema_any_of>(*child).children) { + add_component(*alt, false); } + } else { + add_component(*child, true); } - return _add_rule(rule_name, - _build_object_rule( - properties, required, name, - schema.contains("additionalProperties") ? schema["additionalProperties"] : json())); } - if ((schema_type.is_null() || schema_type == "object" || schema_type == "string") && schema.contains("allOf")) { - std::unordered_set<std::string> required; - std::vector<std::pair<std::string, json>> properties; - std::map<std::string, size_t> enum_values; - const std::string& hybrid_name = name; - std::function<void(const json &, bool)> add_component = [&](const json & comp_schema, bool is_required) { - if (comp_schema.contains("$ref")) { - add_component(_refs[comp_schema["$ref"]], is_required); - } else if (comp_schema.contains("properties")) { - for (const auto & prop : comp_schema["properties"].items()) { - properties.emplace_back(prop.key(), prop.value()); - if (is_required) { - required.insert(prop.key()); - } - } - } else if (comp_schema.contains("enum")) { - for (const auto & v : comp_schema["enum"]) { - const auto rule = _generate_constant_rule(v); - if (enum_values.find(rule) == enum_values.end()) { - enum_values[rule] = 0; - } - enum_values[rule] += 1; - } - } else { - // todo warning + if (!enum_values.empty()) { + std::vector<std::string> enum_intersection; + for (const auto & p : enum_values) { + if (p.second == schema.children.size()) { + enum_intersection.push_back(p.first); } - }; - for (const auto & t : schema["allOf"]) { - if (t.contains("anyOf")) { - for (const auto & tt : t["anyOf"]) { - add_component(tt, false); - } - } else { - add_component(t, true); + } + if (!enum_intersection.empty()) { + return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")"); + } + } + return _add_rule(rule_name, _build_object_rule(properties, required, name, nullptr)); + } + + std::string visit(const common_chat_schema & schema, const std::string & name) { + std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name; + std::string sub_name = name + (name.empty() ? "" : "-"); + + switch (schema.kind()) { + case common_chat_schema::KIND_REF: + return _add_rule(rule_name, _resolve_ref(as<common_chat_schema_ref>(schema))); + case common_chat_schema::KIND_ANY_OF: + return _add_rule(rule_name, _generate_union_rule(name, as<common_chat_schema_any_of>(schema).children)); + case common_chat_schema::KIND_ALL_OF: + return _visit_all_of(as<common_chat_schema_all_of>(schema), name, rule_name); + case common_chat_schema::KIND_CONST: + return _add_rule(rule_name, _generate_constant_rule(as<common_chat_schema_const>(schema).value)); + case common_chat_schema::KIND_ENUM: { + std::vector<std::string> enum_values; + for (const auto & v : as<common_chat_schema_enum>(schema).values) { + enum_values.push_back(_generate_constant_rule(v)); } + return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")"); } - if (!enum_values.empty()) { - std::vector<std::string> enum_intersection; - for (const auto & p : enum_values) { - if (p.second == schema["allOf"].size()) { - enum_intersection.push_back(p.first); - } + case common_chat_schema::KIND_OBJECT: { + const auto & obj = as<common_chat_schema_object>(schema); + if (obj.properties.empty() && obj.additional_properties && obj.additional_properties->kind() == common_chat_schema::KIND_ANY) { + return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object"))); } - if (!enum_intersection.empty()) { - return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")"); + std::vector<std::pair<std::string, const common_chat_schema *>> properties; + std::unordered_set<std::string> required; + for (const auto & prop : obj.properties) { + properties.emplace_back(prop.name, prop.schema.get()); + if (prop.required) { + required.insert(prop.name); + } } + return _add_rule(rule_name, _build_object_rule(properties, required, name, obj.additional_properties.get())); } - return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json())); - } - if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) { - json items = schema.contains("items") ? schema["items"] : schema["prefixItems"]; - if (items.is_array()) { + case common_chat_schema::KIND_TUPLE: { + const auto & items = as<common_chat_schema_tuple>(schema).items; std::string rule = "\"[\" space "; for (size_t i = 0; i < items.size(); i++) { if (i > 0) { rule += " \",\" space "; } - rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i)); + rule += visit(*items[i], sub_name + "tuple-" + std::to_string(i)); } rule += " space \"]\""; return _add_rule(rule_name, rule); } - std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item"); - int min_items = schema.contains("minItems") ? schema["minItems"].get<int>() : 0; - json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json(); - int max_items = max_items_json.is_number_integer() ? max_items_json.get<int>() : std::numeric_limits<int>::max(); - - return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " space \"]\""); - } - if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) { - return _visit_pattern(schema["pattern"], rule_name); - } - if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) { - return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid")); - } - if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) { - auto prim_name = schema_format + "-string"; - return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name))); - } - if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) { - std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); - int min_len = schema.contains("minLength") ? schema["minLength"].get<int>() : 0; - int max_len = schema.contains("maxLength") ? schema["maxLength"].get<int>() : std::numeric_limits<int>::max(); - return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\""); - } - if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) { - int64_t min_value = std::numeric_limits<int64_t>::min(); - int64_t max_value = std::numeric_limits<int64_t>::max(); - if (schema.contains("minimum")) { - min_value = schema["minimum"].get<int64_t>(); - } else if (schema.contains("exclusiveMinimum")) { - min_value = schema["exclusiveMinimum"].get<int64_t>() + 1; + case common_chat_schema::KIND_ARRAY: { + const auto & arr = as<common_chat_schema_array>(schema); + if (arr.items->kind() == common_chat_schema::KIND_ANY && arr.min_items == 0 && arr.max_items < 0) { + return _visit_primitive(rule_name, "array"); + } + std::string item_rule_name = visit(*arr.items, sub_name + "item"); + int max_items = arr.max_items < 0 ? std::numeric_limits<int>::max() : arr.max_items; + return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, arr.min_items, max_items, "\",\" space") + " space \"]\""); } - if (schema.contains("maximum")) { - max_value = schema["maximum"].get<int64_t>(); - } else if (schema.contains("exclusiveMaximum")) { - max_value = schema["exclusiveMaximum"].get<int64_t>() - 1; + case common_chat_schema::KIND_STRING: { + const auto & str = as<common_chat_schema_string>(schema); + if (!str.pattern.empty()) { + return _visit_pattern(str.pattern, rule_name); + } + if (str.format == common_chat_schema::FORMAT_UUID) { + return _visit_primitive(rule_name, "uuid"); + } + if (str.format != common_chat_schema::FORMAT_NONE) { + std::string prim_name = std::string(str.format == common_chat_schema::FORMAT_DATE ? "date" : str.format == common_chat_schema::FORMAT_TIME ? "time" : "date-time") + "-string"; + return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name))); + } + if (str.min_length > 0 || str.max_length >= 0) { + std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); + int max_len = str.max_length < 0 ? std::numeric_limits<int>::max() : str.max_length; + return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, str.min_length, max_len) + " \"\\\"\""); + } + return _visit_primitive(rule_name, "string"); } - std::stringstream out; - out << "("; - build_min_max_int(min_value, max_value, out); - out << ")"; - return _add_rule(rule_name, out.str()); - } - if (schema.empty() || schema_type == "object") { - return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object"))); - } - if (schema_type.is_null() && schema.is_object()) { - // No type constraint and no recognized structural keywords (e.g. {"description": "..."}). - // Per JSON Schema semantics this is equivalent to {} and accepts any value. - return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value"))); - } - if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get<std::string>()) == PRIMITIVE_RULES.end()) { - _errors.push_back("Unrecognized schema: " + schema.dump()); - return ""; + case common_chat_schema::KIND_INTEGER: { + const auto & i = as<common_chat_schema_integer>(schema); + if (i.minimum == std::numeric_limits<int64_t>::min() && i.maximum == std::numeric_limits<int64_t>::max()) { + return _visit_primitive(rule_name, "integer"); + } + std::stringstream out; + out << "("; + build_min_max_int(i.minimum, i.maximum, out); + out << ")"; + return _add_rule(rule_name, out.str()); + } + case common_chat_schema::KIND_NUMBER: + return _visit_primitive(rule_name, "number"); + case common_chat_schema::KIND_BOOLEAN: + return _visit_primitive(rule_name, "boolean"); + case common_chat_schema::KIND_NULL: + return _visit_primitive(rule_name, "null"); + case common_chat_schema::KIND_ANY: + return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value"))); } - // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero - return _add_primitive(rule_name == "root" ? "root" : schema_type.get<std::string>(), PRIMITIVE_RULES.at(schema_type.get<std::string>())); + return ""; } void check_errors() { @@ -1106,134 +990,6 @@ class common_schema_converter { } }; -// common_schema_info implementation (pimpl) - -common_schema_info::common_schema_info() - : impl_(std::make_unique<common_schema_converter>( - [](const std::string &) { return json(); }, - false)) {} - -common_schema_info::~common_schema_info() = default; - -common_schema_info::common_schema_info(common_schema_info &&) noexcept = default; -common_schema_info & common_schema_info::operator=(common_schema_info &&) noexcept = default; - -void common_schema_info::resolve_refs(common_json & schema) { - impl_->resolve_refs(schema, ""); -} - -// Determines if a JSON schema can resolve to a string type through any path. -// Some models emit raw string values rather than JSON-encoded strings for string parameters. -// If any branch of the schema (via oneOf, anyOf, $ref, etc.) permits a string, this returns -// true, allowing callers to handle the value as a raw string for simplicity. -bool common_schema_info::resolves_to_string(const common_json & schema) { - std::unordered_set<std::string> visited_refs; - - std::function<bool(const json &)> check = [&](const json & s) -> bool { - if (!s.is_object()) { - return false; - } - - // Handle $ref - if (s.contains("$ref")) { - const std::string & ref = s["$ref"]; - if (visited_refs.find(ref) != visited_refs.end()) { - // Circular reference, assume not a string to be safe - return false; - } - visited_refs.insert(ref); - auto it = impl_->_refs.find(ref); - if (it != impl_->_refs.end()) { - return check(it->second); - } - return false; - } - - // Check type field - if (s.contains("type")) { - const json & schema_type = s["type"]; - if (schema_type.is_string()) { - if (schema_type == "string") { - return true; - } - } else if (schema_type.is_array()) { - // Type can be an array like ["string", "null"] - for (const auto & t : schema_type) { - if (t == "string") { - return true; - } - } - } - } - - // Check oneOf/anyOf - if any alternative can be a string - if (s.contains("oneOf")) { - for (const auto & alt : s["oneOf"]) { - if (check(alt)) { - return true; - } - } - } - if (s.contains("anyOf")) { - for (const auto & alt : s["anyOf"]) { - if (check(alt)) { - return true; - } - } - } - - // Check allOf - all components must be compatible with string type - if (s.contains("allOf")) { - bool all_string = true; - for (const auto & component : s["allOf"]) { - if (!check(component)) { - all_string = false; - break; - } - } - if (all_string) { - return true; - } - } - - // Check const - if the constant value is a string - if (s.contains("const")) { - if (s["const"].is_string()) { - return true; - } - } - - // Check enum - if any enum value is a string - if (s.contains("enum")) { - for (const auto & val : s["enum"]) { - if (val.is_string()) { - return true; - } - } - } - - // String-specific keywords imply string type - if (s.contains("pattern") || s.contains("minLength") || s.contains("maxLength")) { - return true; - } - - // Check format - many formats imply string - if (s.contains("format")) { - const std::string & fmt = s["format"]; - if (fmt == "date" || fmt == "time" || fmt == "date-time" || - fmt == "uri" || fmt == "email" || fmt == "hostname" || - fmt == "ipv4" || fmt == "ipv6" || fmt == "uuid" || - fmt.find("uuid") == 0) { - return true; - } - } - - return false; - }; - - return check(schema); -} - std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) { #ifdef LLAMA_USE_LLGUIDANCE if (!force_gbnf) { @@ -1242,25 +998,29 @@ std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) #else (void)force_gbnf; #endif // LLAMA_USE_LLGUIDANCE - return build_grammar([&](const common_grammar_builder & callbacks) { - auto copy = schema; - callbacks.resolve_refs(copy); - callbacks.add_schema("", copy); - }); + try { + return json_schema_to_grammar(common_chat_schema_from_json(schema)); + } catch (const std::runtime_error & e) { + throw std::invalid_argument(std::string("JSON schema conversion failed:\n") + e.what()); + } +} + +std::string json_schema_to_grammar(const common_chat_schema_document & schema) { + common_chat_schema_converter converter(false); + converter.visit(*schema.root, ""); + converter.check_errors(); + return converter.format_grammar(); } std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options) { - common_schema_converter converter([&](const std::string &) { return json(); }, options.dotall); + common_chat_schema_converter converter(options.dotall); common_grammar_builder builder { /* .add_rule = */ [&](const std::string & name, const std::string & rule) { return converter._add_rule(name, rule); }, - /* .add_schema = */ [&](const std::string & name, const common_json & schema) { - return converter.visit(schema, name == "root" ? "" : name); + /* .add_schema = */ [&](const std::string & name, const common_chat_schema & schema) { + return converter.add_schema(name == "root" ? "" : name, schema); }, - /* .resolve_refs = */ [&](common_json & schema) { - converter.resolve_refs(schema, ""); - } }; cb(builder); converter.check_errors(); diff --git a/common/json-schema-to-grammar.h b/common/json-schema-to-grammar.h index 84ed71c76a13..b928c250bbef 100644 --- a/common/json-schema-to-grammar.h +++ b/common/json-schema-to-grammar.h @@ -1,37 +1,17 @@ #pragma once +#include "json-schema.h" #include "json.h" #include <functional> -#include <memory> #include <string> -std::string json_schema_to_grammar(const common_json & schema, - bool force_gbnf = false); - -class common_schema_converter; - -// Probes a JSON schema to extract information about its structure and type constraints. -class common_schema_info { - std::unique_ptr<common_schema_converter> impl_; - - public: - common_schema_info(); - ~common_schema_info(); - - common_schema_info(const common_schema_info &) = delete; - common_schema_info & operator=(const common_schema_info &) = delete; - common_schema_info(common_schema_info &&) noexcept; - common_schema_info & operator=(common_schema_info &&) noexcept; - - void resolve_refs(common_json & schema); - bool resolves_to_string(const common_json & schema); -}; +std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf = false); +std::string json_schema_to_grammar(const common_chat_schema_document & schema); struct common_grammar_builder { - std::function<std::string(const std::string &, const std::string &)> add_rule; - std::function<std::string(const std::string &, const common_json &)> add_schema; - std::function<void(common_json &)> resolve_refs; + std::function<std::string(const std::string &, const std::string &)> add_rule; + std::function<std::string(const std::string &, const common_chat_schema &)> add_schema; }; struct common_grammar_options { diff --git a/common/json-schema.cpp b/common/json-schema.cpp new file mode 100644 index 000000000000..6898840e7d1f --- /dev/null +++ b/common/json-schema.cpp @@ -0,0 +1,514 @@ +#include "json-schema.h" +#include "common.h" + +#include <cmath> +#include <map> +#include <stdexcept> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +class common_chat_schema_builder { + const common_json & root_; + common_chat_schema_document & doc_; + + // the targets built here, moved into doc_ once the whole schema is built + std::map<std::string, common_chat_schema_ptr> refs_; + + // ref nodes get their target once every $ref is built, a cycle would otherwise need it too early + std::vector<common_chat_schema_ref *> pending_; + + [[noreturn]] static void fail(const std::string & path, const std::string & msg) { + throw std::runtime_error("JSON schema error at " + path + ": " + msg); + } + + static int get_count(const common_json & schema, const std::string & key, const std::string & path, int def) { + if (!schema.contains(key)) { + return def; + } + const common_json & value = schema.at(key); + if (!value.is_number_integer() || value.get<int>() < 0) { + fail(path, key + " must be a non-negative integer"); + } + return value.get<int>(); + } + + // a fractional bound is rounded inwards, towards the integers it still admits + static int64_t get_bound(const common_json & schema, const std::string & key, const std::string & path, bool round_up) { + const common_json & value = schema.at(key); + if (value.is_number_integer()) { + return value.get<int64_t>(); + } + if (!value.is_number()) { + fail(path, key + " must be a number"); + } + double d = value.get<double>(); + return (int64_t) (round_up ? std::ceil(d) : std::floor(d)); + } + + static common_chat_schema::string_format get_format(const common_json & schema, const std::string & path) { + if (!schema.contains("format")) { + return common_chat_schema::FORMAT_NONE; + } + const common_json & value = schema.at("format"); + if (!value.is_string()) { + fail(path, "format must be a string"); + } + std::string format = value.get<std::string>(); + if (format == "date") { + return common_chat_schema::FORMAT_DATE; + } + if (format == "time") { + return common_chat_schema::FORMAT_TIME; + } + if (format == "date-time") { + return common_chat_schema::FORMAT_DATE_TIME; + } + if (format == "uuid" || (format.size() == 5 && format.compare(0, 4, "uuid") == 0 && format[4] >= '1' && format[4] <= '5')) { + return common_chat_schema::FORMAT_UUID; + } + return common_chat_schema::FORMAT_NONE; + } + + const common_json & resolve_ref(const std::string & ref, const std::string & path) { + const common_json * target = &root_; + auto tokens = string_split(ref.substr(1), "/"); + for (size_t i = 1; i < tokens.size(); i++) { + const std::string & sel = tokens[i]; + if (target->is_object() && target->contains(sel)) { + target = &target->at(sel); + } else if (target->is_array()) { + size_t idx; + try { + idx = std::stoull(sel); + } catch (const std::logic_error &) { + idx = target->size(); + } + if (idx >= target->size()) { + fail(path, "cannot resolve $ref " + ref + ", " + sel + " is out of range"); + } + target = &target->at(idx); + } else { + fail(path, "cannot resolve $ref " + ref + ", " + sel + " not found"); + } + } + return *target; + } + + common_chat_schema_ptr build_ref(const common_json & value, const std::string & path) { + if (!value.is_string()) { + fail(path, "$ref must be a string"); + } + std::string ref = value.get<std::string>(); + if (ref.compare(0, 2, "#/") != 0) { + fail(path, "unsupported $ref " + ref + ", only references into the same document are supported"); + } + if (refs_.find(ref) == refs_.end()) { + // reserve the key first, so that a cycle back to this $ref stops here + refs_[ref] = nullptr; + refs_[ref] = build_node(resolve_ref(ref, path), ref); + } + auto node = std::make_unique<common_chat_schema_ref>(ref); + pending_.push_back(node.get()); + return node; + } + + template <typename T> + common_chat_schema_ptr build_alternatives(const common_json & alts, const std::string & path) { + if (!alts.is_array()) { + fail(path, "must be an array of schemas"); + } + if (alts.empty()) { + fail(path, "must not be empty"); + } + auto node = std::make_unique<T>(); + size_t i = 0; + for (const auto & alt : alts) { + node->children.push_back(build_node(alt, path + "/" + std::to_string(i++))); + } + return node; + } + + common_chat_schema_ptr build_object(const common_json & schema, const std::string & path) { + auto node = std::make_unique<common_chat_schema_object>(); + + std::unordered_set<std::string> required; + if (schema.contains("required") && schema.at("required").is_array()) { + for (const auto & name : schema.at("required")) { + if (name.is_string()) { + required.insert(name.get<std::string>()); + } + } + } + + if (schema.contains("properties")) { + const common_json & properties = schema.at("properties"); + if (!properties.is_object()) { + fail(path, "properties must be an object"); + } + for (const auto & [name, prop] : properties.items()) { + node->properties.push_back({name, build_node(prop, path + "/properties/" + name), required.count(name) > 0}); + } + } + + if (schema.contains("additionalProperties")) { + const common_json & additional = schema.at("additionalProperties"); + if (additional.is_boolean()) { + if (additional.get<bool>()) { + node->additional_properties = std::make_unique<common_chat_schema_any>(); + } + } else if (additional.is_object()) { + node->additional_properties = build_node(additional, path + "/additionalProperties"); + } else { + fail(path, "additionalProperties must be a boolean or a schema"); + } + } else if (!schema.contains("properties")) { + // {"type": "object"} on its own accepts any object + node->additional_properties = std::make_unique<common_chat_schema_any>(); + } + + return node; + } + + common_chat_schema_ptr build_array(const common_json & schema, const std::string & path) { + auto node = std::make_unique<common_chat_schema_array>(); + if (schema.contains("items") || schema.contains("prefixItems")) { + // "items" wins when both are present; as in the converter, a schema instead of an array is the item schema + const std::string key = schema.contains("items") ? "items" : "prefixItems"; + const common_json & items = schema.at(key); + if (items.is_array()) { + auto tuple = std::make_unique<common_chat_schema_tuple>(); + size_t i = 0; + for (const auto & item : items) { + tuple->items.push_back(build_node(item, path + "/" + key + "/" + std::to_string(i++))); + } + return tuple; + } + node->items = build_node(items, path + "/" + key); + } else { + node->items = std::make_unique<common_chat_schema_any>(); + } + node->min_items = get_count(schema, "minItems", path, 0); + node->max_items = get_count(schema, "maxItems", path, -1); + return node; + } + + common_chat_schema_ptr build_string(const common_json & schema, const std::string & path) { + auto node = std::make_unique<common_chat_schema_string>(); + if (schema.contains("pattern")) { + const common_json & pattern = schema.at("pattern"); + if (!pattern.is_string()) { + fail(path, "pattern must be a string"); + } + node->pattern = pattern.get<std::string>(); + } + node->format = get_format(schema, path); + node->min_length = get_count(schema, "minLength", path, 0); + node->max_length = get_count(schema, "maxLength", path, -1); + return node; + } + + common_chat_schema_ptr build_integer(const common_json & schema, const std::string & path) { + auto node = std::make_unique<common_chat_schema_integer>(); + if (schema.contains("minimum")) { + node->minimum = get_bound(schema, "minimum", path, /* round_up */ true); + } else if (schema.contains("exclusiveMinimum")) { + node->minimum = get_bound(schema, "exclusiveMinimum", path, /* round_up */ false) + 1; + } + if (schema.contains("maximum")) { + node->maximum = get_bound(schema, "maximum", path, /* round_up */ false); + } else if (schema.contains("exclusiveMaximum")) { + node->maximum = get_bound(schema, "exclusiveMaximum", path, /* round_up */ true) - 1; + } + return node; + } + + common_chat_schema_ptr build_node(const common_json & schema, const std::string & path) { + if (!schema.is_object()) { + fail(path, "schema must be an object"); + } + if (schema.contains("$ref")) { + return build_ref(schema.at("$ref"), path); + } + if (schema.contains("oneOf") || schema.contains("anyOf")) { + const std::string key = schema.contains("oneOf") ? "oneOf" : "anyOf"; + return build_alternatives<common_chat_schema_any_of>(schema.at(key), path + "/" + key); + } + + common_json type; + if (schema.contains("type")) { + type = schema.at("type"); + } + if (type.is_array()) { + // {"type": ["a", "b"], ...} is {"anyOf": [{"type": "a", ...}, {"type": "b", ...}]} + if (type.empty()) { + fail(path, "type must not be empty"); + } + auto node = std::make_unique<common_chat_schema_any_of>(); + size_t i = 0; + for (const auto & t : type) { + common_json alt = schema; + alt["type"] = t; + node->children.push_back(build_node(alt, path + "/type/" + std::to_string(i++))); + } + return node; + } + if (schema.contains("const")) { + return std::make_unique<common_chat_schema_const>(schema.at("const")); + } + if (schema.contains("enum")) { + const common_json & values = schema.at("enum"); + if (!values.is_array() || values.empty()) { + fail(path, "enum must be a non-empty array"); + } + auto node = std::make_unique<common_chat_schema_enum>(); + for (const auto & value : values) { + node->values.push_back(value); + } + return node; + } + if (!type.is_null() && !type.is_string()) { + fail(path, "type must be a string or an array of strings"); + } + + const std::string type_name = type.is_string() ? type.get<std::string>() : ""; + const bool has_properties = schema.contains("properties") || + (schema.contains("additionalProperties") && schema.at("additionalProperties") != true); + + if (type_name.empty()) { + // without a type the structural keywords decide, in the same order as the converter + if (has_properties) { + return build_object(schema, path); + } + if (schema.contains("allOf")) { + return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf"); + } + if (schema.contains("items") || schema.contains("prefixItems")) { + return build_array(schema, path); + } + if (schema.contains("pattern") || schema.contains("minLength") || schema.contains("maxLength") || get_format(schema, path) != common_chat_schema::FORMAT_NONE) { + return build_string(schema, path); + } + return std::make_unique<common_chat_schema_any>(); + } + if (type_name == "object") { + if (!has_properties && schema.contains("allOf")) { + return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf"); + } + return build_object(schema, path); + } + if (type_name == "string") { + if (schema.contains("allOf")) { + return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf"); + } + return build_string(schema, path); + } + if (type_name == "array") { + return build_array(schema, path); + } + if (type_name == "integer") { + return build_integer(schema, path); + } + if (type_name == "number") { + return std::make_unique<common_chat_schema_number>(); + } + if (type_name == "boolean") { + return std::make_unique<common_chat_schema_boolean>(); + } + if (type_name == "null") { + return std::make_unique<common_chat_schema_null>(); + } + fail(path, "unrecognized type " + type_name); + } + + public: + common_chat_schema_builder(const common_json & root, common_chat_schema_document & doc) : root_(root), doc_(doc) {} + + common_chat_schema_ptr build() { + auto node = build_node(root_, "#"); + for (auto & entry : refs_) { + doc_.refs[entry.first] = std::move(entry.second); + } + for (auto * ref : pending_) { + ref->target = doc_.refs.at(ref->ref).get(); + } + return node; + } +}; + +common_chat_schema_document common_chat_schema_from_json(const common_json & schema) { + common_chat_schema_document doc; + doc.root = common_chat_schema_builder(schema, doc).build(); + return doc; +} + +static common_chat_schema::value_type json_type(const common_json & value) { + if (value.is_null()) { + return common_chat_schema::TYPE_NULL; + } + if (value.is_boolean()) { + return common_chat_schema::TYPE_BOOLEAN; + } + if (value.is_number_integer()) { + return common_chat_schema::TYPE_INTEGER; + } + if (value.is_number()) { + return common_chat_schema::TYPE_NUMBER; + } + if (value.is_string()) { + return common_chat_schema::TYPE_STRING; + } + if (value.is_array()) { + return common_chat_schema::TYPE_ARRAY; + } + return common_chat_schema::TYPE_OBJECT; +} + +static common_chat_schema::type_set value_types_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) { + switch (s.kind()) { + case common_chat_schema::KIND_ANY: + return common_chat_schema::type_set::all(); + case common_chat_schema::KIND_NULL: + return { common_chat_schema::TYPE_NULL }; + case common_chat_schema::KIND_BOOLEAN: + return { common_chat_schema::TYPE_BOOLEAN }; + case common_chat_schema::KIND_NUMBER: + return { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER }; + case common_chat_schema::KIND_INTEGER: + return { common_chat_schema::TYPE_INTEGER }; + case common_chat_schema::KIND_STRING: + return { common_chat_schema::TYPE_STRING }; + case common_chat_schema::KIND_ARRAY: + case common_chat_schema::KIND_TUPLE: + return { common_chat_schema::TYPE_ARRAY }; + case common_chat_schema::KIND_OBJECT: + return { common_chat_schema::TYPE_OBJECT }; + case common_chat_schema::KIND_CONST: + return { json_type(static_cast<const common_chat_schema_const &>(s).value) }; + case common_chat_schema::KIND_ENUM: { + common_chat_schema::type_set types; + for (const auto & value : static_cast<const common_chat_schema_enum &>(s).values) { + types.add(json_type(value)); + } + return types; + } + case common_chat_schema::KIND_REF: { + const auto * target = static_cast<const common_chat_schema_ref &>(s).target; + if (!target || !visited.insert(target).second) { + // a cycle contributes no type, to be safe + return {}; + } + auto types = value_types_impl(*target, visited); + visited.erase(target); + return types; + } + case common_chat_schema::KIND_ANY_OF: { + common_chat_schema::type_set types; + for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) { + types |= value_types_impl(*child, visited); + } + return types; + } + case common_chat_schema::KIND_ALL_OF: { + auto types = common_chat_schema::type_set::all(); + for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) { + types &= value_types_impl(*child, visited); + } + return types; + } + } + return {}; +} + +common_chat_schema::type_set common_chat_schema::value_types() const { + std::unordered_set<const common_chat_schema *> visited; + return value_types_impl(*this, visited); +} + +static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) { + switch (s.kind()) { + case common_chat_schema::KIND_STRING: + return true; + case common_chat_schema::KIND_CONST: + return static_cast<const common_chat_schema_const &>(s).value.is_string(); + case common_chat_schema::KIND_ENUM: + for (const auto & v : static_cast<const common_chat_schema_enum &>(s).values) { + if (v.is_string()) { + return true; + } + } + return false; + case common_chat_schema::KIND_REF: { + // a cycle is taken as not a string, to be safe + const auto * target = static_cast<const common_chat_schema_ref &>(s).target; + if (!target || !visited.insert(target).second) { + return false; + } + bool result = may_be_string_impl(*target, visited); + visited.erase(target); + return result; + } + case common_chat_schema::KIND_ANY_OF: + for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) { + if (may_be_string_impl(*child, visited)) { + return true; + } + } + return false; + case common_chat_schema::KIND_ALL_OF: { + // every child must allow a string, an any child constrains nothing + bool any_string = false; + for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) { + if (child->kind() == common_chat_schema::KIND_ANY) { + continue; + } + if (!may_be_string_impl(*child, visited)) { + return false; + } + any_string = true; + } + return any_string; + } + default: + return false; + } +} + +bool common_chat_schema::may_be_string() const { + std::unordered_set<const common_chat_schema *> visited; + return may_be_string_impl(*this, visited); +} + +const char * common_chat_schema::kind_name(node_kind kind) { + switch (kind) { + case KIND_ANY: return "any"; + case KIND_REF: return "ref"; + case KIND_ANY_OF: return "anyOf"; + case KIND_ALL_OF: return "allOf"; + case KIND_CONST: return "const"; + case KIND_ENUM: return "enum"; + case KIND_NULL: return "null"; + case KIND_BOOLEAN: return "boolean"; + case KIND_NUMBER: return "number"; + case KIND_INTEGER: return "integer"; + case KIND_STRING: return "string"; + case KIND_ARRAY: return "array"; + case KIND_TUPLE: return "tuple"; + case KIND_OBJECT: return "object"; + } + return "?"; +} + +const char * common_chat_schema::type_name(value_type type) { + switch (type) { + case TYPE_NULL: return "null"; + case TYPE_BOOLEAN: return "boolean"; + case TYPE_NUMBER: return "number"; + case TYPE_INTEGER: return "integer"; + case TYPE_STRING: return "string"; + case TYPE_ARRAY: return "array"; + case TYPE_OBJECT: return "object"; + } + return "?"; +} diff --git a/common/json-schema.h b/common/json-schema.h new file mode 100644 index 000000000000..084208c96201 --- /dev/null +++ b/common/json-schema.h @@ -0,0 +1,198 @@ +#pragma once + +#include "json.h" + +#include <cstdint> +#include <initializer_list> +#include <map> +#include <memory> +#include <string> +#include <vector> + +// JSON schema, covering the subset that json_schema_to_grammar() can convert. + +struct common_chat_schema { + enum node_kind { + KIND_ANY, + KIND_REF, + KIND_ANY_OF, + KIND_ALL_OF, + KIND_CONST, + KIND_ENUM, + KIND_NULL, + KIND_BOOLEAN, + KIND_NUMBER, + KIND_INTEGER, + KIND_STRING, + KIND_ARRAY, + KIND_TUPLE, + KIND_OBJECT, + }; + + enum value_type { + TYPE_NULL, + TYPE_BOOLEAN, + TYPE_NUMBER, + TYPE_INTEGER, + TYPE_STRING, + TYPE_ARRAY, + TYPE_OBJECT, + }; + + enum string_format { + FORMAT_NONE, + FORMAT_UUID, // uuid, uuid1 .. uuid5 + FORMAT_DATE, + FORMAT_TIME, + FORMAT_DATE_TIME, + }; + + class type_set { + uint32_t mask_ = 0; + + public: + type_set() = default; + type_set(std::initializer_list<value_type> types) { + for (auto type : types) { + add(type); + } + } + + static type_set all() { + return { TYPE_NULL, TYPE_BOOLEAN, TYPE_NUMBER, TYPE_INTEGER, TYPE_STRING, TYPE_ARRAY, TYPE_OBJECT }; + } + + void add(value_type type) { mask_ |= 1u << type; } + + bool has(value_type type) const { return (mask_ & (1u << type)) != 0; } + bool is_only(value_type type) const { return mask_ == (1u << type); } + bool empty() const { return mask_ == 0; } + + type_set & operator|=(const type_set & other) { mask_ |= other.mask_; return *this; } + type_set & operator&=(const type_set & other) { mask_ &= other.mask_; return *this; } + + bool operator==(const type_set & other) const { return mask_ == other.mask_; } + bool operator!=(const type_set & other) const { return mask_ != other.mask_; } + }; + + virtual ~common_chat_schema() = default; + virtual node_kind kind() const = 0; + + type_set value_types() const; + + // Whether a value matching the schema may be a string, through any branch of it. + bool may_be_string() const; + + static const char * kind_name(node_kind kind); + static const char * type_name(value_type type); +}; + +using common_chat_schema_ptr = std::unique_ptr<common_chat_schema>; + +struct common_chat_schema_any : common_chat_schema { + node_kind kind() const override { return KIND_ANY; } +}; + +// {"$ref": "#/..."}, only references into the same document are supported +struct common_chat_schema_ref : common_chat_schema { + std::string ref; + const common_chat_schema * target = nullptr; // owned by common_chat_schema_document::refs + + explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {} + + node_kind kind() const override { return KIND_REF; } +}; + +// oneOf / anyOf, or a "type" array expanded to one alternative per type +struct common_chat_schema_any_of : common_chat_schema { + std::vector<common_chat_schema_ptr> children; + + node_kind kind() const override { return KIND_ANY_OF; } +}; + +struct common_chat_schema_all_of : common_chat_schema { + std::vector<common_chat_schema_ptr> children; + + node_kind kind() const override { return KIND_ALL_OF; } +}; + +struct common_chat_schema_const : common_chat_schema { + common_json value; + + explicit common_chat_schema_const(common_json value) : value(std::move(value)) {} + + node_kind kind() const override { return KIND_CONST; } +}; + +struct common_chat_schema_enum : common_chat_schema { + std::vector<common_json> values; + + node_kind kind() const override { return KIND_ENUM; } +}; + +struct common_chat_schema_null : common_chat_schema { + node_kind kind() const override { return KIND_NULL; } +}; + +struct common_chat_schema_boolean : common_chat_schema { + node_kind kind() const override { return KIND_BOOLEAN; } +}; + +struct common_chat_schema_number : common_chat_schema { + node_kind kind() const override { return KIND_NUMBER; } +}; + +// bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in +struct common_chat_schema_integer : common_chat_schema { + int64_t minimum = INT64_MIN; // INT64_MIN for unbounded + int64_t maximum = INT64_MAX; // INT64_MAX for unbounded + + node_kind kind() const override { return KIND_INTEGER; } +}; + +struct common_chat_schema_string : common_chat_schema { + std::string pattern; // empty when absent + string_format format = FORMAT_NONE; + int min_length = 0; + int max_length = -1; // -1 for unbounded + + node_kind kind() const override { return KIND_STRING; } +}; + +struct common_chat_schema_array : common_chat_schema { + common_chat_schema_ptr items; // a common_chat_schema_any when "items" is absent + int min_items = 0; + int max_items = -1; // -1 for unbounded + + node_kind kind() const override { return KIND_ARRAY; } +}; + +struct common_chat_schema_tuple : common_chat_schema { + std::vector<common_chat_schema_ptr> items; + + node_kind kind() const override { return KIND_TUPLE; } +}; + +struct common_chat_schema_property { + std::string name; + common_chat_schema_ptr schema; + bool required = false; +}; + +struct common_chat_schema_object : common_chat_schema { + std::vector<common_chat_schema_property> properties; // in schema order + common_chat_schema_ptr additional_properties; // null when not allowed + + node_kind kind() const override { return KIND_OBJECT; } +}; + +struct common_chat_schema_document { + common_chat_schema_ptr root; + std::map<std::string, common_chat_schema_ptr> refs; +}; + +// A document shared by the PEG parsers built from its nodes, which it keeps alive +using common_chat_schema_document_ptr = std::shared_ptr<const common_chat_schema_document>; + +// Throws std::runtime_error when the schema falls outside the supported subset. +common_chat_schema_document common_chat_schema_from_json(const common_json & schema); diff --git a/common/json.h b/common/json.h index f3ad4edee8b0..95b6d6f212e5 100644 --- a/common/json.h +++ b/common/json.h @@ -82,6 +82,9 @@ struct common_json_value { // note: a nested pair {"a", "b"} does not build, use common_json::array({"a", "b"}) for an array common_json_value(std::initializer_list<common_json_item> items); + template <typename T, typename std::enable_if<std::is_enum<T>::value, int>::type = 0> + common_json_value(T val) : common_json_value((typename std::underlying_type<T>::type) val) {} + template <typename T, typename std::enable_if<std::is_integral<T>::value && !std::is_same<T, bool>::value, int>::type = 0> common_json_value(T val) : type(std::is_signed<T>::value ? VAL_INT : VAL_UINT) { if (std::is_signed<T>::value) { @@ -111,6 +114,7 @@ struct common_json_item { // the types common_json_value holds on its own // anything else reaches its common_json ctor and recurses forever template <typename T> struct common_json_is_value : std::integral_constant<bool, + std::is_enum<T>::value || std::is_arithmetic<T>::value || std::is_same<T, std::nullptr_t>::value || std::is_same<T, std::string>::value || diff --git a/common/log.cpp b/common/log.cpp index 0f0cb7902c8a..0a9a4eb9ea49 100644 --- a/common/log.cpp +++ b/common/log.cpp @@ -1,5 +1,6 @@ #include "common.h" #include "log.h" +#include "json.h" #include <chrono> #include <condition_variable> @@ -36,6 +37,16 @@ void common_log_set_verbosity_thold(int verbosity) { common_log_verbosity_thold = verbosity; } +static bool common_log_jsonl = false; + +bool common_log_get_jsonl(void) { + return common_log_jsonl; +} + +void common_log_set_jsonl(bool jsonl) { + common_log_jsonl = jsonl; +} + static int64_t t_us() { return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count(); } @@ -66,6 +77,17 @@ static const char* g_col[] = { "", }; +static const char * level_str(enum ggml_log_level level) { + switch (level) { + case GGML_LOG_LEVEL_DEBUG: return "debug"; + case GGML_LOG_LEVEL_INFO: return "info"; + case GGML_LOG_LEVEL_WARN: return "warn"; + case GGML_LOG_LEVEL_ERROR: return "error"; + case GGML_LOG_LEVEL_CONT: return "cont"; + default: return "none"; + } +} + struct common_log_entry { enum ggml_log_level level {GGML_LOG_LEVEL_INFO}; @@ -74,6 +96,8 @@ struct common_log_entry { int64_t timestamp { 0 }; bool is_end { false }; // signals the worker thread to stop bool prefix { false }; + bool jsonl { false }; + bool is_json { false }; // msg already holds a serialized JSON object common_log_entry(size_t size = 256) : msg(size) { } @@ -88,11 +112,29 @@ struct common_log_entry { fcur = stdout; - if (level != GGML_LOG_LEVEL_NONE) { + if (level != GGML_LOG_LEVEL_NONE && !jsonl) { fcur = stderr; } } + if (jsonl) { + if (is_json) { + fprintf(fcur, "%s\n", msg.data()); + fflush(fcur); + return; + } + + common_json obj = { + {"type", "log"}, + {"time", timestamp}, + {"level", level_str(level)}, + {"msg", msg.data()}, + }; + fprintf(fcur, "%s\n", obj.dump_safe().c_str()); + fflush(fcur); + return; + } + if (level != GGML_LOG_LEVEL_NONE && level != GGML_LOG_LEVEL_CONT && prefix) { if (timestamp) { // [M.s.ms.us] @@ -246,6 +288,8 @@ struct common_log { entry.is_end = false; entry.level = level; entry.prefix = prefix; + entry.jsonl = common_log_jsonl; + entry.is_json = false; entry.timestamp = 0; if (timestamps) { entry.timestamp = t_us() - t_start; @@ -255,6 +299,42 @@ struct common_log { cv_new.notify_one(); } + void add_json(const char * type, const common_json & obj) { + const common_json full = { + {"type", type}, + {"data", obj}, + }; + + const std::string text = full.dump_safe(); + + std::unique_lock<std::mutex> lock(mtx); + + // block if the queue is full + cv_full.wait(lock, [this]() { return !running || !is_full(); }); + + if (!running) { + // discard messages while the worker thread is paused + return; + } + + auto & entry = queue[tail]; + + if (entry.msg.size() < text.size() + 1) { + entry.msg.resize(text.size() + 1); + } + memcpy(entry.msg.data(), text.c_str(), text.size() + 1); + + entry.is_end = false; + entry.level = GGML_LOG_LEVEL_NONE; + entry.prefix = false; + entry.jsonl = true; + entry.is_json = true; + entry.timestamp = 0; + + tail = (tail + 1) % queue.size(); + cv_new.notify_one(); + } + void resume() { std::lock_guard<std::mutex> lock(mtx); @@ -406,6 +486,14 @@ void common_log_add(struct common_log * log, enum ggml_log_level level, const ch va_end(args); } +void common_log_add_json(struct common_log * log, const char * type, const common_json & obj) { + if (!common_log_jsonl) { + return; + } + + log->add_json(type, obj); +} + void common_log_set_file(struct common_log * log, const char * file) { log->set_file(file); } diff --git a/common/log.h b/common/log.h index f033582526ad..e36b09463e60 100644 --- a/common/log.h +++ b/common/log.h @@ -43,6 +43,10 @@ int common_log_get_verbosity_thold(void); void common_log_set_verbosity_thold(int verbosity); // not thread-safe +bool common_log_get_jsonl(void); + +void common_log_set_jsonl(bool jsonl); // not thread-safe + int common_log_get_verbosity(enum ggml_log_level level); void common_log_default_callback(enum ggml_log_level level, const char * text, void * user_data); @@ -126,3 +130,17 @@ void common_log_flush (struct common_log * log); // f #define LOG_WRNV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_WARN, verbosity, __VA_ARGS__) #define LOG_ERRV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_ERROR, verbosity, __VA_ARGS__) #define LOG_CNTV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_CONT, verbosity, __VA_ARGS__) + +class common_json; // defined in common/json.h + +// helper allows different types of json output +// no-op if --log-jsonl is not set +void common_log_add_json(struct common_log * log, const char * type, const common_json & data); + +// will only print if --log-jsonl is set +#define LOG_JSON(type, data) \ + do { \ + if (common_log_get_jsonl()) { \ + common_log_add_json(common_log_main(), type, data); \ + } \ + } while (0) diff --git a/common/parsers/cohere2moe.cpp b/common/parsers/cohere2moe.cpp new file mode 100644 index 000000000000..59595368dc3e --- /dev/null +++ b/common/parsers/cohere2moe.cpp @@ -0,0 +1,141 @@ +#include "parsers.h" + +// Cohere2 MoE (a.k.a. "North Code") parser. +// +// The assistant turn is fully marker-wrapped: +// <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> +// <|START_THINKING|>{reasoning}<|END_THINKING|> +// then EITHER content: <|START_TEXT|>{content}<|END_TEXT|> +// OR tool calls: <|START_ACTION|>[ +// {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... +// ]<|END_ACTION|> +// <|END_OF_TURN_TOKEN|> +// +// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is +// the template default), so the model's output continues from *inside* the thinking block. The +// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix +// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself, +// regardless of whether they came from the generation prompt or the generated text. +common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + const std::string TURN_START = "<|START_OF_TURN_TOKEN|>"; + const std::string TURN_END = "<|END_OF_TURN_TOKEN|>"; + const std::string CHATBOT = "<|CHATBOT_TOKEN|>"; + const std::string USER = "<|USER_TOKEN|>"; + const std::string SYSTEM = "<|SYSTEM_TOKEN|>"; + const std::string THINK_START = "<|START_THINKING|>"; + const std::string THINK_END = "<|END_THINKING|>"; + const std::string TEXT_START = "<|START_TEXT|>"; + const std::string TEXT_END = "<|END_TEXT|>"; + const std::string ACTION_START = "<|START_ACTION|>"; + const std::string ACTION_END = "<|END_ACTION|>"; + const std::string RESULT_START = "<|START_TOOL_RESULT|>"; + const std::string RESULT_END = "<|END_TOOL_RESULT|>"; + + // Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker. + const std::string GEN_PREFIX = TURN_START + CHATBOT; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = {THINK_END}; + data.preserved_tokens = { + TURN_START, TURN_END, CHATBOT, USER, SYSTEM, + THINK_START, THINK_END, + TEXT_START, TEXT_END, + ACTION_START, ACTION_END, + RESULT_START, RESULT_END, + }; + + // Declare per-role message delimiters. Tool results are rendered with the + // system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before + // the plain "system" one (it is a strict superset, and the role split tries delimiters in order). + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX }, + { COMMON_CHAT_ROLE_USER, TURN_START + USER }, + { COMMON_CHAT_ROLE_TOOL, TURN_START + SYSTEM + RESULT_START }, + { COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + TEXT_START + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PREFIX); + auto end = p.end(); + + // The thinking block is always present (the generation prompt forces <|START_THINKING|>). + // When extracting reasoning, capture its body; otherwise keep the whole block (markers + // included) inline as content, matching reasoning_format=NONE conventions. + common_peg_parser reasoning = p.eps(); + if (extract_reasoning) { + reasoning = p.optional(p.literal(THINK_START) + + p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) + + p.optional(p.literal(THINK_END))); + } else { + reasoning = p.optional(p.content(p.literal(THINK_START) + + p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) + + p.optional(p.literal(THINK_END)))); + } + + auto text_content = has_response_format + ? p.literal(TEXT_START) + + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + + p.optional(p.literal(TEXT_END)) + : p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END)); + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end; + } + + auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + + // <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|> + auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls, + /* force_tool_calls = */ true, + /* name_key = */ "tool_name", + /* args_key = */ "parameters", + /* array_wrapped = */ true, + /* function_is_key = */ false, + /* call_id_key = */ "", + /* gen_call_id_key = */ "tool_call_id", + /* parameters_order = */ { "tool_call_id", "tool_name", "parameters" }); + + // Content and tool calls are mutually exclusive in this format. + common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content }); + + return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START } + }; + } + + return data; +} diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp new file mode 100644 index 000000000000..e00132a7ca8d --- /dev/null +++ b/common/parsers/deepseek.cpp @@ -0,0 +1,282 @@ +#include "parsers.h" + +// The DeepSeek V4 reference implementation renders consecutive tool results into a single +// user block, ordered by the tool call order of the preceding assistant message (matched +// by tool call id) rather than by the order they appear in the conversation. +static json deepseek_v4_sort_tool_results(const json & messages) { + json adjusted = messages; + std::map<std::string, size_t> call_order; + + for (size_t i = 0; i < adjusted.size();) { + const auto & msg = adjusted[i]; + const auto role = msg.value("role", ""); + + if (role == "assistant" && msg.contains("tool_calls") && + msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) { + call_order.clear(); + const auto & tool_calls = msg.at("tool_calls"); + for (size_t idx = 0; idx < tool_calls.size(); idx++) { + auto id = tool_calls[idx].value("id", ""); + if (!id.empty()) { + call_order[id] = idx; + } + } + i++; + continue; + } + + if (role != "user" && role != "tool") { + i++; + continue; + } + + // collect a maximal run of user/tool messages - they render into one user block + std::vector<size_t> tool_positions; + size_t run_end = i; + for (; run_end < adjusted.size(); run_end++) { + const auto r = adjusted[run_end].value("role", ""); + if (r == "tool") { + tool_positions.push_back(run_end); + } else if (r != "user") { + break; + } + } + + if (tool_positions.size() > 1 && !call_order.empty()) { + std::vector<json> results; + results.reserve(tool_positions.size()); + for (auto pos : tool_positions) { + results.push_back(adjusted[pos]); + } + std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) { + const auto order = [&](const json & m) { + auto it = call_order.find(m.value("tool_call_id", "")); + return it == call_order.end() ? (size_t) 0 : it->second; + }; + return order(a) < order(b); + }); + for (size_t k = 0; k < tool_positions.size(); k++) { + adjusted[tool_positions[k]] = std::move(results[k]); + } + } + + i = run_end; + } + + return adjusted; +} + +common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + // V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls" + // instead of "function_calls", renders tool results in tool call order and its + // non-thinking generation prompt ends with a bare </think> instead of an empty + // <think></think> pair. + const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos; + // V4.1 keeps the V4 format but its DSML tag names carry a leading space: + // "<｜DSML｜ calls>", "<｜DSML｜ invoke>", "<｜DSML｜ parameter>" (the release's + // encoding/encoding.py: tool_calls_block_name = " calls"). The template builds the + // opener as "<' + dsml_token + ' calls>" (single or double quotes), so that literal + // identifies a V4.1 template. + const bool is_v41 = is_v4 && (tmpl.source().find("' calls>") != std::string::npos || + tmpl.source().find("\" calls>") != std::string::npos); + + std::optional<json> adjusted_messages; + if (is_v4) { + adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages); + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + std::optional<json> additional_context; + if (is_v4 && has_response_format) { + additional_context = json{ { "response_format", inputs.json_schema } }; + } + + const std::string DSML = "｜DSML｜"; + const std::string THINK_START = "<think>"; + const std::string THINK_END = "</think>"; + const std::string TC_BLOCK = is_v41 ? " calls" : is_v4 ? "tool_calls" : "function_calls"; + const std::string INVOKE_TAG = is_v41 ? " invoke" : "invoke"; + const std::string PARAM_TAG = is_v41 ? " parameter" : "parameter"; + const std::string FC_START = "<" + DSML + TC_BLOCK + ">"; + const std::string FC_END = "</" + DSML + TC_BLOCK + ">"; + const std::string INVOKE_START = "<" + DSML + INVOKE_TAG; + const std::string INVOKE_END = "</" + DSML + INVOKE_TAG + ">"; + const std::string PARAM_START = "<" + DSML + PARAM_TAG; + const std::string PARAM_END = "</" + DSML + PARAM_TAG + ">"; + const std::string GEN_PROMPT = "<｜Assistant｜>"; + const std::string TC_SEPARATOR = "\n\n"; + + // lets the server find user turns in the prompt and place context checkpoints there + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, GEN_PROMPT }, + { COMMON_CHAT_ROLE_USER, "<｜User｜>" }, + }; + + data.prompt = common_chat_template_direct_apply_impl( + tmpl, inputs, adjusted_messages, std::nullopt, additional_context); + data.generation_prompt = common_chat_template_generation_prompt_impl( + tmpl, inputs, adjusted_messages, std::nullopt, additional_context); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = {THINK_END, FC_START}; + data.preserved_tokens = { + DSML, + THINK_START, + THINK_END, + }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + if (is_v4 && msg.reasoning_content.empty()) { + data.generation_prompt = GEN_PROMPT + THINK_END; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += msg.render_content(); + } + } else { + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } + } + + data.prompt += data.generation_prompt; + } + + bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PROMPT); + auto end = p.end(); + + // build tool call section first since we might need it in reasoning + auto tool_choice = p.choice(); + if (has_tool_calls) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + + std::vector<common_peg_parser> required_parsers; + std::vector<common_peg_parser> optional_parsers; + foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) { + bool is_string = param.schema->may_be_string(); + + auto arg = p.tool_arg( + p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param.name)) + + p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) + + (is_string ? + p.tool_arg_string_value(p.until(PARAM_END)) : + p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param.name + "-schema", + doc, *param.schema))) + + p.tool_arg_close(p.literal(PARAM_END))); + + auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg); + if (param.required) { + required_parsers.push_back(named_arg); + } else { + optional_parsers.push_back(named_arg); + } + }); + + common_peg_parser args_seq = p.eps(); + for (size_t i = 0; i < required_parsers.size(); i++) { + if (i > 0) { + args_seq = args_seq + p.space(); + } + args_seq = args_seq + required_parsers[i]; + } + + if (!optional_parsers.empty()) { + common_peg_parser any_opt = p.choice(); + for (const auto & opt : optional_parsers) { + any_opt |= opt; + } + args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1); + } + + common_peg_parser invoke_body = args_seq; + auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") + + p.tool_name(p.literal(name)) + p.literal("\">\n")) + + invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END))); + + tool_choice |= p.rule("tool-" + name, func_parser); + }); + } + + common_peg_parser tool_calls = p.eps(); + if (inputs.parallel_tool_calls) { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + + p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); + } else { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); + } + + auto reasoning = p.eps(); + auto reasoning_with_tc = p.eps(); + auto obligatory_tool_calls = tool_calls; + bool allow_reasoning_with_tc = false; + + if (!require_tools) { + tool_calls = p.optional(tool_calls); + } + + if (extract_reasoning && inputs.enable_thinking) { + reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); + reasoning_with_tc = THINK_START + + p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) + + p.space() + obligatory_tool_calls; + allow_reasoning_with_tc = true; + } else if (extract_reasoning) { + // Thinking disabled but reasoning extraction requested: the generation prompt + // contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that + // must still be consumed. + reasoning = is_v4 + ? p.optional(p.literal(THINK_END)) + : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END)); + } + + if (has_response_format) { + auto response_format = p.rule("response-format", + p.literal("```json") + p.space() + + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + + p.space() + p.literal("```")); + return generation_prompt + reasoning + response_format + end; + } + + if (!has_tool_calls) { + return generation_prompt + reasoning + p.content(p.rest()) + end; + } + + auto content_before_tools = p.negate(p.literal(THINK_START)) + + p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) + + p.space(); + return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end : + generation_prompt + reasoning + content_before_tools + tool_calls + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = has_tools && !require_tools; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START }, + }; + } + + return data; +} diff --git a/common/parsers/functionary-v3-2.cpp b/common/parsers/functionary-v3-2.cpp new file mode 100644 index 000000000000..9d47f0a3328f --- /dev/null +++ b/common/parsers/functionary-v3-2.cpp @@ -0,0 +1,96 @@ +#include "parsers.h" + +// Functionary v3.2 - uses recipient-based format: >>>recipient\n{content} +common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.preserved_tokens = { + ">>>all", + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + data.generation_prompt = "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\n" + msg.render_content(); + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + // Functionary v3.2 format: + // - Normal content: >>>all\n{content} + // - Tool calls: >>>function_name\n{json_args} + // Generation prompt ends with ">>>" so model outputs recipient immediately + + // Build content parser for >>>all\n{content} + // When tools are present, content stops before the next ">>>" (tool call) + // When no tools, content goes until end + auto content_until_tool = p.literal("all\n") + p.content(p.until(">>>")); + auto content_until_end = p.literal("all\n") + p.content(p.rest()); + auto generation_prompt = p.literal("<|start_header_id|>assistant<|end_header_id|>\n\n>>>"); + + // If no tools or tool_choice is NONE, just parse content + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + // When no tools, just match the prefix and capture everything after + return generation_prompt + content_until_end + p.end(); + } + + // Build tool call parsers for each available function + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const auto schema = common_chat_tool_parameters(function); + + // Tool format: >>>function_name\n{json_args} + auto tool_parser = p.tool( + p.tool_open(p.tool_name(p.literal(name)) + p.literal("\n")) + + p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) + ); + + tool_choice |= p.rule("tool-" + name, tool_parser); + }); + + auto content_only = content_until_end; + auto tools_only = p.trigger_rule("tools", p.one_or_more(tool_choice)); + auto content_and_tools = content_until_tool + tools_only; + + auto ret = p.eps(); + if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { + if (inputs.parallel_tool_calls) { + ret = p.choice({ content_and_tools, tools_only }) + p.end(); + } else { + ret = p.choice({ content_until_tool + tool_choice, tools_only }) + p.end(); + } + } else if (inputs.parallel_tool_calls) { + ret = p.choice({ content_and_tools, content_only, tools_only }) + p.end(); + } else { + auto content_and_tool = content_until_tool + tool_choice; + ret = p.choice({ content_and_tool, content_only, tool_choice }) + p.end(); + } + return generation_prompt + ret; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + // Grammar trigger for when the model starts outputting a tool call + // (after the initial ">>>" in the generation prompt but recipient other than "all") + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, ">>>(?!all)" } + }; + } + + return data; +} diff --git a/common/parsers/gemma4.cpp b/common/parsers/gemma4.cpp new file mode 100644 index 000000000000..f43c5ae46dbf --- /dev/null +++ b/common/parsers/gemma4.cpp @@ -0,0 +1,307 @@ +#include "parsers.h" + +namespace workaround { + +// Gemma4 uses a custom tool_responses field instead of role:tool messages. +// +// This will transform a sequence of messages: +// assistant(tool_call+) -> tool+ -> assistant(content) +// +// Into a single assistant message containing a tool_responses field: +// assistant(content + tool_call + tool_responses) +// +// This is necessary for the Gemma4 chat template to properly format the prompt. +// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4 +struct gemma4_model_turn_builder { + json & messages; + size_t pos; + json tool_calls = json::array(); + json tool_responses = json::array(); + json content; + json reasoning_content; + + gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {} + + void collect() { + // Collect the first assistant message + auto & msg = messages[pos]; + if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { + // According to the prompt formatting guide, we need to preserve reasoning_content + // between function calls. The current chat templates do not support this, but we will do it anyway. + reasoning_content = msg.at("reasoning_content"); + } + for (auto & tc : msg.at("tool_calls")) { + tool_calls.push_back(tc); + } + pos++; + + // Collect tool call results + while (pos < messages.size() && messages[pos].value("role", "") == "tool") { + collect_result(messages[pos]); + pos++; + } + + // Check if the next assistant message is the final message + if (pos < messages.size() && messages[pos].value("role", "") == "assistant") { + auto & next = messages[pos]; + if (!has_tool_calls(next) && has_content(next)) { + content = next.at("content"); + pos++; + } + } + } + + void collect_result(const json & curr) { + json response; + if (curr.contains("content")) { + const auto & content = curr.at("content"); + if (content.is_string()) { + // Try to parse the content as JSON; fall back to raw string + try { + response = json::parse(content.get<std::string>()); + } catch (...) { + response = content; + } + } else { + response = content; + } + } + + std::string name; + + // Match name with corresponding tool call + size_t idx = tool_responses.size(); + if (idx < tool_calls.size()) { + auto & tc = tool_calls[idx]; + if (tc.contains("function")) { + name = tc.at("function").value("name", ""); + } + } + + // Fallback to the tool call id + if (name.empty()) { + name = curr.value("tool_call_id", ""); + } + + tool_responses.push_back({{"name", name}, {"response", response}}); + } + + json build() { + collect(); + + json msg = { + {"role", "assistant"}, + {"tool_calls", tool_calls}, + }; + if (!tool_responses.empty()) { + msg["tool_responses"] = tool_responses; + } + if (!content.is_null()) { + msg["content"] = content; + } + if (!reasoning_content.is_null()) { + msg["reasoning_content"] = reasoning_content; + } + return msg; + } + + static bool has_content(const json & msg) { + if (!msg.contains("content") || msg.at("content").is_null()) { + return false; + } + const auto & content = msg.at("content"); + if (content.is_string() && !content.get<std::string>().empty()) { + return true; + } + if (content.is_array() && !content.empty()) { + return true; + } + return false; + } + + static bool has_tool_calls(const json & msg) { + return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty(); + } +}; + +void convert_tool_responses_gemma4(json & messages) { + json result = json::array(); + size_t i = 0; + + while (i < messages.size()) { + auto & msg = messages[i]; + + if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") || + !msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) { + result.push_back(msg); + i++; + continue; + } + + gemma4_model_turn_builder builder(messages, i); + result.push_back(builder.build()); + i = builder.pos; + } + + messages = result; +} + +} + +common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + + if (inputs.add_generation_prompt && string_ends_with(data.prompt, "<turn|>\n")) { + // This may happen if the model generates content + tool_call, the + // template does not add the model's next turn and confuses the model + // from emitting its proper reasoning token sequence. + data.generation_prompt = "<|turn>model\n"; + data.prompt += data.generation_prompt; + } + + data.message_delimiters = { + { COMMON_CHAT_ROLE_USER, "<|turn>user" }, + { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" }, + }; + + data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4; + data.supports_thinking = true; + data.thinking_start_tag = "<|channel>thought"; + data.thinking_end_tags = {"<channel|>"}; + + data.preserved_tokens = { + "<|channel>", + "<channel|>", + "<|tool_call>", + "<tool_call|>", + "<|turn>", + }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = string_ends_with(data.prompt, "<turn|>\n") ? "<|turn>model\n" : ""; + data.generation_prompt += "<|channel>thought\n" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "<channel|>" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto start = p.rule("start", p.optional(p.literal("<|turn>model\n"))); + + if (extract_reasoning) { + p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>")); + } else { + p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("<channel|>") + p.literal("<channel|>"))); + } + + auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), ""); + auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>")); + + if (has_response_format) { + auto response_format = p.literal("```json") << + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) << + p.literal("```"); + return start + p.optional(thought) + response_format; + } + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + // Gemma4 tool calling syntax + // Rules should match traversal logic in gemma4_to_json() + p.rule("gemma4-string-content", p.until("<|\"|>")); + p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>")); + p.rule("gemma4-bool", p.json_bool()); + p.rule("gemma4-null", p.json_null()); + p.rule("gemma4-number", p.json_number()); + p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":")); + p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value")); + p.rule("gemma4-dict", [&]() { + auto ws = p.space(); + auto member = p.ref("gemma4-dict-kv"); + auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))}); + return p.sequence({ + p.literal("{"), ws, + p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})}) + }); + }); + p.rule("gemma4-array", [&]() { + auto ws = p.space(); + auto value = p.ref("gemma4-value"); + auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))}); + return p.sequence({ + p.literal("["), ws, + p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})}) + }); + }); + p.rule("gemma4-value", [&]() { + return p.choice({ + p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"), + p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null") + }); + }); + + auto tool_choice = p.choice(); + + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules + // const auto & params = function.at("parameters"); + + tool_choice |= p.rule("tool-" + name, p.tool(p.sequence({ + p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))), + p.tool_args(p.ref("gemma4-dict")), + }))); + }); + + auto tool_call = p.trigger_rule("tool-call", p.repeat( + "<|tool_call>call:" + tool_choice + "<tool_call|>", + /* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0, + /* max = */ inputs.parallel_tool_calls ? -1 : 1 + )); + + if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { + return start + thought + tool_call; + } + + auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>")); + auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>", "<|tool_call>"}))); + auto message = p.rule("message", thought + content); + return start + p.zero_or_more(message) + scan_to_toolcall + tool_call; + } + + // Gemma 4 may emit an extra <|channel>thought\n<channel|> at the end of the content. It may + // also emit a single trailing <channel|> token. Consume all complete reasoning blocks and + // then stop at the first unmatched <channel|> token. + auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>"}))); + auto message = p.rule("message", thought + content); + return start + p.one_or_more(message); + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" }, + }; + } + + return data; +} diff --git a/common/parsers/gigachat-v3.cpp b/common/parsers/gigachat-v3.cpp new file mode 100644 index 000000000000..48abcb3f33c1 --- /dev/null +++ b/common/parsers/gigachat-v3.cpp @@ -0,0 +1,76 @@ +#include "parsers.h" + +common_chat_params common_chat_params_init_gigachat_v3( + const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = false; + data.preserved_tokens = { + "<|message_sep|>\n\n", + "<|role_sep|>\n", + }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + data.generation_prompt = "assistant<|role_sep|>\n" + msg.render_content(); + data.prompt += data.generation_prompt; + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n"; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto ret = p.eps(); + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + // Build a choice of all available tools + auto tool_choice = p.choice(); + for (const auto & tool : inputs.tools) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const auto schema = common_chat_tool_parameters(function); + + auto tool_name = p.json_member("name", "\"" + p.tool_name(p.literal(name)) + "\""); + auto tool_args = p.json_member("arguments", p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))); + + auto tool_open = p.tool_open(p.literal("{") << tool_name); + + tool_choice |= p.rule("tool-" + name, tool_open << "," << tool_args << "}"); + } + + // Define the tool call structure + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + auto max_calls = 1; // parallel toolcalls are not supported + auto tool_call = p.rule("tool-call", p.literal(tool_call_start_prefix) + tool_choice); + auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(tool_call, /* min = */ min_calls, /* max = */ max_calls)); + + ret = p.content(p.until("<|message_sep|>\n\n")) << tool_calls; + } else { + // Content only parser + include_grammar = false; + ret = p.content(p.rest()); + } + + return p.literal("assistant<|role_sep|>\n") + ret; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + {COMMON_GRAMMAR_TRIGGER_TYPE_WORD, tool_call_start_prefix} + }; + } + return data; +} diff --git a/common/parsers/gpt-oss.cpp b/common/parsers/gpt-oss.cpp new file mode 100644 index 000000000000..00beb41a47ee --- /dev/null +++ b/common/parsers/gpt-oss.cpp @@ -0,0 +1,158 @@ +#include "parsers.h" + +common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + // Copy reasoning to the "thinking" field as expected by the gpt-oss template + auto adjusted_messages = json::array(); + for (auto msg : inputs.messages) { + if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { + msg["thinking"] = msg.at("reasoning_content"); + if (msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) { + msg.erase("content"); + } + } + adjusted_messages.push_back(msg); + } + + auto prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override= */ adjusted_messages); + + // Check if we need to replace the return token with end token during + // inference and without generation prompt. For more details see: + // https://github.com/ggml-org/llama.cpp/issues/15417 + if (inputs.is_inference && !inputs.add_generation_prompt) { + static constexpr std::string_view return_token = "<|return|>"; + static constexpr std::string_view end_token = "<|end|>"; + if (size_t pos = prompt.rfind(return_token); pos != std::string::npos) { + prompt.replace(pos, return_token.length(), end_token); + } + } + + data.prompt = prompt; + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages); + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, + { COMMON_CHAT_ROLE_USER, "<|start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, + { COMMON_CHAT_ROLE_TOOL, "<|start|>functions" }, + }; + + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + + data.thinking_start_tag = "<|channel|>analysis<|message|>"; + data.thinking_end_tags = {"<|end|>"}; + + // These special tokens are required to parse properly, so we include them + // even if parse_tool_calls is false. + data.preserved_tokens = { + "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>", + }; + + // Adjust prompt for continuation + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto start = p.rule("start", p.literal("<|start|>assistant")); + auto end = p.rule("end", p.literal("<|end|>")); + auto content = p.rule("message-content", p.until("<|end|>")); + auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis")); + auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1); + + // Occasionally, gpt-oss-20b will prefix channels with this commentary + auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant"))); + auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>"); + + if (extract_reasoning) { + p.rule("analysis", start_analysis + p.reasoning(content) + end); + } else { + p.rule("analysis", p.content(start_analysis + content + end)); + } + + auto analysis = p.ref("analysis"); + auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end); + auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content)); + + // Consume any unsolicited tool calls, e.g. builtin functions + auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end)); + + auto any = p.rule("any", preamble | analysis); + + if (has_response_format) { + auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); + auto response_format = p.rule("response-format", + p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema))); + + return p.zero_or_more(start + analysis) + start + response_format; + } + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto tool_choice = p.choice(); + + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const auto params = common_chat_tool_parameters(function); + + auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name)); + auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); + auto args = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", params)); + + // recipient in role header + // <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS + auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args); + + // recipient in channel header + // <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS + auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args); + + tool_choice |= p.rule("tool-" + name, tool_in_role | tool_in_channel); + }); + + auto tool_call = p.trigger_rule("tool-call", tool_choice); + + if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { + return p.zero_or_more(start + any) + start + tool_call; + } + + return p.zero_or_more(start + any) + start + (tool_call | final_msg); + } + + return p.zero_or_more(start + any) + start + (final_msg | unsolicited); + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" }, + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" }, + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" }, + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" } + }; + } + + return data; +} diff --git a/common/parsers/kimi-k2.cpp b/common/parsers/kimi-k2.cpp new file mode 100644 index 000000000000..5ee9121abab2 --- /dev/null +++ b/common/parsers/kimi-k2.cpp @@ -0,0 +1,128 @@ +#include "parsers.h" + +// Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index> +// The ID contains both the function name and an incrementing counter +common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.preserved_tokens = { + "<|tool_calls_section_begin|>", + "<|tool_calls_section_end|>", + "<|tool_call_begin|>", + "<|tool_call_argument_begin|>", + "<|tool_call_end|>", + "<think>", + "</think>", + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + const std::string SECTION_BEGIN = "<|tool_calls_section_begin|>"; + const std::string SECTION_END = "<|tool_calls_section_end|>"; + const std::string CALL_BEGIN = "<|tool_call_begin|>"; + const std::string ARGS_BEGIN = "<|tool_call_argument_begin|>"; + const std::string CALL_END = "<|tool_call_end|>"; + + const std::string THINK_START = "<think>"; + const std::string THINK_END = "</think>"; + const std::string GEN_PROMPT = "<|im_assistant|>assistant<|im_middle|>"; + + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = {THINK_END}; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + // Kimi K2 Thinking format: + // - Reasoning: <think>{reasoning}</think> + // - Content: text after reasoning + // - Tool calls section: + // <|tool_calls_section_begin|> + // <|tool_call_begin|>functions.<name>:<index><|tool_call_argument_begin|>{json_args}<|tool_call_end|> + // ... + // <|tool_calls_section_end|> + // The ID format is: functions.<function_name>:<counter> where counter is 0, 1, 2, ... + + // Tool call markers + auto end = p.end(); + + // Note: this model is CRAZY. It can diverge from its supposed tool calling pattern in so many ways it's not funny. + // For example, it can call tools at the end of reasoning without closing reasoning... + auto reasoning = extract_reasoning ? p.optional(THINK_START + p.reasoning( + p.until_one_of({ THINK_END, "<|tool_calls_section_begin|>", "<|tool_call_begin|>" })) + + p.optional(p.literal(THINK_END))) : p.eps(); + auto generation_prompt = p.literal(GEN_PROMPT); + + + // Content only parser (no tools) + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return generation_prompt + reasoning + p.content(p.rest()) + end; + } + + // Build tool call parsers for each available function + // The ID format is: functions.<name>:<index> + // We need to match: functions.<name>:<digits> + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const auto schema = common_chat_tool_parameters(function); + + // Match: functions.<name>:<digits> + // Capture the full call id (functions.<name>:<digits>) using tool_id tag + auto tool_id = p.tool_id(p.literal("functions.") + p.tool_name(p.literal(name)) + p.literal(":") + p.chars("[0-9]", 1, -1)); + auto tool_parser = p.tool( + p.tool_open(tool_id + p.literal(ARGS_BEGIN)) + + p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) + + p.tool_close(p.optional((p.literal(CALL_END)))) + ); + + tool_choice |= p.rule("tool-" + name, tool_parser); + }); + + // Tool calls section: <|tool_calls_section_begin|> tool_calls <|tool_calls_section_end|> + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + auto max_calls = inputs.parallel_tool_calls ? -1 : 1; + // Use trigger_rule so grammar generator knows where to start generating rules + auto tool_calls = p.rule("tool-calls", + p.optional(p.literal(SECTION_BEGIN)) + + p.trigger_rule("tool-call", p.repeat(CALL_BEGIN + tool_choice, min_calls, max_calls) + + p.optional(p.literal(SECTION_END))) + ); + + auto content_before_tools = p.content(p.until_one_of({ SECTION_BEGIN, CALL_BEGIN })); + + return generation_prompt + reasoning + content_before_tools + tool_calls + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call_begin|>" } + }; + } + + return data; +} diff --git a/common/parsers/kimi-k3.cpp b/common/parsers/kimi-k3.cpp new file mode 100644 index 000000000000..989e39f956f0 --- /dev/null +++ b/common/parsers/kimi-k3.cpp @@ -0,0 +1,167 @@ +#include "parsers.h" + +// Kimi K3 - XTML tagged format, built by open_tag/close_tag macros: +// open_tag(t, attrs) = <|open|>t k="v"...<|sep|> close_tag(t) = <|close|>t<|sep|> +// assistant := [think] [response] [tools] close_tag(message) <|end_of_msg|> +// the generation prompt already opens the think (or response) section, so the +// section opener is optional here - same as Kimi K2 Thinking +common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + + const std::string SEP = "<|sep|>"; + const std::string MSG_START = "<|open|>message role=\"assistant\"<|sep|>"; + const std::string THINK_START = "<|open|>think<|sep|>"; + const std::string THINK_END = "<|close|>think<|sep|>"; + const std::string RESP_START = "<|open|>response<|sep|>"; + const std::string RESP_END = "<|close|>response<|sep|>"; + const std::string TOOLS_START = "<|open|>tools<|sep|>"; + const std::string TOOLS_END = "<|close|>tools<|sep|>"; + const std::string CALL_START = "<|open|>call tool=\""; + const std::string CALL_END = "<|close|>call<|sep|>"; + const std::string ARG_START = "<|open|>argument key=\""; + const std::string ARG_END = "<|close|>argument<|sep|>"; + const std::string MSG_END = "<|close|>message<|sep|>"; + const std::string EOM_TOKEN = "<|end_of_msg|>"; + + // only the markers are special tokens. tag names ("think", "response", ...) are + // normal tokens and must not be preserved, or prose with those words is broken + data.preserved_tokens = { + "<|open|>", + "<|close|>", + "<|sep|>", + "<|end_of_msg|>", + }; + + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = { THINK_END }; + + // per-role message-start delimiters. user/assistant messages only have the role + // attribute, so the full opener is used. system and tool messages have more + // attributes, so those delimiters stop after the closing quote of the role + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|open|>message role=\"assistant\"<|sep|>" }, + { COMMON_CHAT_ROLE_USER, "<|open|>message role=\"user\"<|sep|>" }, + { COMMON_CHAT_ROLE_TOOL, "<|open|>message role=\"tool\"" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|open|>message role=\"system\"" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = MSG_START + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + RESP_START + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto end = p.end(); + + auto start = p.optional(p.literal(MSG_START)); + + // the think section is always consumed, even with reasoning extraction off: + // the generation prompt ends with open_tag('think'), so it is always present. + // reasoning stops at its own closer, or at the response opener if the model + // skips the closer + auto think_body = extract_reasoning ? p.reasoning(p.until_one_of({ THINK_END, RESP_START })) : + p.content(p.until_one_of({ THINK_END, RESP_START })); + + auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body + + p.optional(p.literal(THINK_END))); + + // content runs to the response closer, or to the next section if truncated + auto response = p.optional(p.literal(RESP_START)) + + p.content(p.until_one_of({ RESP_END, TOOLS_START, MSG_END })) + + p.optional(p.literal(RESP_END)); + + // the EOG token after the message closer reaches the parser as text, + // so it must be consumed or the parse stays incomplete + auto trailer = p.optional(p.literal(MSG_END)) + p.optional(p.literal(EOM_TOKEN)); + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return start + reasoning + response + trailer + end; + } + + auto tool_choices = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const json schema = common_chat_tool_parameters(function); + + // arguments come one tag per key, with the JSON type in a type="..." + // attribute. the type is taken from the tool schema instead, as it tells + // us if the value is JSON or a literal string + auto args = p.eps(); + if (schema.contains("properties") && !schema.at("properties").empty()) { + auto arg_choices = p.choice(); + for (const auto & prop : schema.at("properties").items()) { + const std::string & key = prop.key(); + + std::string type = "string"; + if (prop.value().is_object() && prop.value().contains("type") && + prop.value().at("type").is_string()) { + type = prop.value().at("type").get<std::string>(); + } + + auto value = type == "string" ? p.tool_arg_string_value(p.until(ARG_END)) : + p.tool_arg_value(p.until(ARG_END)); + + // skip the trailing type="..." attribute: anything up to <|sep|> + arg_choices |= p.rule("kimi-k3-arg-" + name + "-" + key, + p.tool_arg(p.tool_arg_open(p.literal(ARG_START)) + + p.tool_arg_name(p.literal(key)) + p.literal("\"") + + p.until(SEP) + p.literal(SEP) + value + + p.tool_arg_close(p.literal(ARG_END)))); + } + args = p.zero_or_more(arg_choices); + } + + // skip the trailing index="N" attribute the same way + auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) + p.literal("\"") + + p.until(SEP) + p.literal(SEP)) + + p.tool_args(args) + p.tool_close(p.literal(CALL_END))); + + tool_choices |= p.rule("kimi-k3-tool-" + name, call); + }); + + // all calls go inside one tools section, then the message is closed. the + // message closer is part of the trigger rule, or else the lazy grammar + // rejects it once tool calls have started + auto tools_section = + p.trigger_rule("kimi-k3-tool-call", p.literal(TOOLS_START) + p.one_or_more(tool_choices) + + p.literal(TOOLS_END) + p.optional(p.literal(MSG_END)) + + p.optional(p.literal(EOM_TOKEN))); + + auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section : + p.optional(tools_section); + + return start + reasoning + response + tools + trailer + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOLS_START }, + }; + } + + return data; +} diff --git a/common/parsers/lfm2.cpp b/common/parsers/lfm2.cpp new file mode 100644 index 000000000000..b662ea50053c --- /dev/null +++ b/common/parsers/lfm2.cpp @@ -0,0 +1,118 @@ +#include "parsers.h" + +// LFM2 format detection: template uses <|tool_list_start|>[...]<|tool_list_end|> around the tool list +// and <|tool_call_start|>[...]<|tool_call_end|> around each tool call +bool is_lfm2_template(const std::string & src) { + return src.find("<|tool_list_start|>") != std::string::npos && + src.find("<|tool_list_end|>") != std::string::npos; +} + +// LFM2/LFM2.5 parser. Tool calls are almost Python-style and parallel-capable +// (except dotted names and JSON literals true/false/null). +// Always wrapped in <|tool_call_start|>[name(args)]<|tool_call_end|> with optional <think> reasoning. +// tool_list_tokens preserves LFM2 system tool-list markers. +common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl, + const autoparser::generation_params & inputs, + bool tool_list_tokens) { + common_chat_params data; + + const std::string TOOL_CALL_START = "<|tool_call_start|>"; + const std::string TOOL_CALL_END = "<|tool_call_end|>"; + const std::string TOOL_LIST_START = "<|tool_list_start|>"; + const std::string TOOL_LIST_END = "<|tool_list_end|>"; + const std::string THINK_START = "<think>"; + const std::string THINK_END = "</think>"; + const std::string GEN_PROMPT = "<|im_start|>assistant\n"; + + // Copy reasoning to the "thinking" field the template expects + auto adjusted_messages = json::array(); + for (auto msg : inputs.messages) { + if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { + msg["thinking"] = msg.at("reasoning_content"); + } + adjusted_messages.push_back(msg); + } + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.preserved_tokens = { TOOL_CALL_START, TOOL_CALL_END, THINK_START, THINK_END }; + if (tool_list_tokens) { + data.preserved_tokens.push_back(TOOL_LIST_START); + data.preserved_tokens.push_back(TOOL_LIST_END); + } + + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = {THINK_END}; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + // Gate by reasoning format and whether the template supports <think> + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE && + tmpl.source().find(THINK_START) != std::string::npos; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PROMPT); + auto end = p.end(); + + auto reasoning = p.eps(); + if (extract_reasoning) { + // LFM2.5's template does not prefill an opening <think> into the generation prompt, + // so the model must emit it itself each turn. It sometimes skips the opening tag and + // dives straight into reasoning, closing only with </think>. Without forced-open + // handling that raw reasoning bleeds into content. Make the opening tag + // optional so a leading reasoning block terminated by </think> is captured whether or + // not <think> was emitted. A pure-content turn has no </think>, so until() fails to + // find the terminator, the sequence fails, and the outer optional collapses -> the + // whole output falls through to content exactly as before. + reasoning = p.optional(p.optional(p.literal(THINK_START)) + p.reasoning(p.until(THINK_END)) + p.literal(THINK_END)); + } + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + if (has_response_format) { + auto response_format = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)); + return generation_prompt + reasoning + response_format + end; + } + return generation_prompt + reasoning + p.content(p.rest()) + end; + } + auto tool_calls = p.rule("tool-calls", + p.trigger_rule("tool-call", + p.literal(TOOL_CALL_START) + + p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true) + + p.literal(TOOL_CALL_END) + ) + ); + + auto content = p.content(p.until(TOOL_CALL_START)); + + return generation_prompt + reasoning + content + tool_calls + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOL_CALL_START } + }; + } + + return data; +} diff --git a/common/parsers/ling3.cpp b/common/parsers/ling3.cpp new file mode 100644 index 000000000000..8b49847e24fc --- /dev/null +++ b/common/parsers/ling3.cpp @@ -0,0 +1,194 @@ +#include "parsers.h" + +// Ling 3.0 / Bailing V3 - <role>X</role> sections with tagged tool calls: +// assistant := [<think> ... </think>] [content] {<tool_call>name +// <arg_key>k</arg_key>\n<arg_value>v</arg_value> ...</tool_call>} +// The generation prompt ends with "<role>ASSISTANT</role>\n<think>", so the model +// never emits the opening think tag, and a tool call can arrive before any +// </think>. Reasoning therefore terminates at the think close tag or at a tool +// call start, like the Qwen3-Coder and Kimi K3 parsers. With thinking off the +// template pre-closes the think block instead, and the model emits bare content. +common_chat_params common_chat_params_init_ling3(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + + const std::string ROLE = "<role>ASSISTANT</role>"; + const std::string THINK_START = "<think>"; + const std::string THINK_END = "</think>"; + const std::string CALL_START = "<tool_call>"; + const std::string CALL_END = "</tool_call>"; + const std::string ARG_KEY = "<arg_key>"; + const std::string ARG_KEY_END = "</arg_key>"; + const std::string ARG_VAL = "<arg_value>"; + const std::string ROLE_END = "<|role_end|>"; + const std::string ARG_VAL_END = "</arg_value>"; + + data.preserved_tokens = { + THINK_START, THINK_END, CALL_START, CALL_END, + ARG_KEY, ARG_KEY_END, ARG_VAL, ARG_VAL_END, ROLE_END, + }; + + data.thinking_start_tag = THINK_START; + // Support both </think> and <tool_call> as reasoning end sequences: a call + // can be emitted before the think block is closed. + data.thinking_end_tags = { THINK_END, CALL_START }; + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<role>ASSISTANT</role>" }, + { COMMON_CHAT_ROLE_USER, "<role>HUMAN</role>" }, + { COMMON_CHAT_ROLE_TOOL, "<role>OBSERVATION</role>" }, + { COMMON_CHAT_ROLE_SYSTEM, "<role>SYSTEM</role>" }, + }; + + // the model may spell the end-of-turn control token out as text tokens, + // which does not stop generation; a literal stop string catches it either + // way (as the Laguna patch does for its </assistant> token) + data.additional_stops = { ROLE_END }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = ROLE + "\n" + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + // The generation prompt pre-opens the think block when thinking is on, so + // the opening tag is optional here and reasoning runs until </think> or a + // tool call start; with thinking off the template pre-closes the block and + // everything the model emits is content. + bool think_open = false; + if (inputs.has_continuation()) { + think_open = inputs.continue_final_message != COMMON_CHAT_CONTINUATION_CONTENT; + } else { + auto last_open = data.generation_prompt.rfind(THINK_START); + auto last_close = data.generation_prompt.rfind(THINK_END); + think_open = last_open != std::string::npos && + (last_close == std::string::npos || last_open > last_close); + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto end = p.end(); + + // the effective parse input is generation_prompt + model output, so the + // assistant opener is optionally consumed here + auto opener = p.optional(p.literal(ROLE) + p.optional(p.space())); + + // the generation prompt pre-opens the think block, so the opening tag + // is optional; a missing close tag does not swallow a tool call + auto body_end = think_open ? p.until_one_of({ THINK_END, CALL_START }) : p.until_one_of({ THINK_END }); + auto think_body = extract_reasoning ? p.reasoning(body_end) : p.content(body_end); + + auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body + + p.optional(p.literal(THINK_END))); + + // content between the think block and the first tool call, plus any + // trailing text after the last tool call, are plain content + auto content = p.optional(p.content(p.until_one_of({ CALL_START }))); + + // a trailing end-of-turn token is consumed instead of leaking into content + auto tail = p.optional(p.content(p.until(ROLE_END))) + p.optional(p.literal(ROLE_END)); + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return opener + reasoning + tail + end; + } + + auto tool_choices = p.choice(); + auto arg_close = p.tool_arg_close(p.literal(ARG_VAL_END)); + auto arg_string = p.rule("ling3-arg-string", + p.tool_arg_string_value(p.until(ARG_VAL_END)) + arg_close); + + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + + std::vector<common_peg_parser> required_args; + std::vector<common_peg_parser> optional_args; + + // each argument may be preceded by whitespace: the model emits + // newlines between arguments, the template history does not + foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) { + auto rule_name = "ling3-arg-" + name + "-" + param.name; + + auto types = param.schema->value_types(); + + // string arguments are raw text up to the closing tag, other + // types parse as JSON per their schema; each alternative + // consumes the closing tag itself so a JSON prefix can not + // commit the choice before the tag matches + auto arg_value = p.eps(); + if (!types.has(common_chat_schema::TYPE_STRING)) { + arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close; + } else if (types.is_only(common_chat_schema::TYPE_STRING)) { + arg_value = arg_string; + } else { + // the parser tries the JSON alternative first to type the value + arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close) | arg_string, + "ling3-arg-string"); + } + + auto arg = p.rule(rule_name, + p.optional(p.space()) + + p.tool_arg(p.tool_arg_open(p.literal(ARG_KEY) + p.tool_arg_name(p.literal(param.name)) + + p.literal(ARG_KEY_END)) + + p.optional(p.space()) + p.literal(ARG_VAL) + + arg_value)); + + (param.required ? required_args : optional_args).push_back(arg); + }); + + // required arguments in any order (as Qwen3-Coder does), then + // optional ones in any order and number + auto args = p.permute("ling3-" + name + "-args", required_args); + if (!optional_args.empty()) { + args = args + p.zero_or_more(p.choice(optional_args)); + } + + auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) + + p.optional(p.space())) + + p.tool_args(args) + + p.tool_close(p.optional(p.space()) + p.literal(CALL_END))); + + tool_choices |= p.rule("ling3-tool-" + name, call); + }); + + auto calls = inputs.parallel_tool_calls ? + tool_choices + p.zero_or_more(p.space() + tool_choices) : + tool_choices; + + auto tools_section = p.trigger_rule("ling3-tool-call", calls + p.space() + + p.optional(p.content(p.until(ROLE_END))) + p.optional(p.literal(ROLE_END))); + + auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section : + p.optional(tools_section); + + return opener + reasoning + content + tools + tail + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, CALL_START }, + }; + } + + return data; +} diff --git a/common/parsers/minicpm5.cpp b/common/parsers/minicpm5.cpp new file mode 100644 index 000000000000..4d18d3d9600e --- /dev/null +++ b/common/parsers/minicpm5.cpp @@ -0,0 +1,130 @@ +#include "parsers.h" + +// MiniCPM5 format: +// - Reasoning: <think>{reasoning}</think> (optional) +// - Tool calls: <function name="foo"><param name="bar">value</param></function> +common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.preserved_tokens = { + "<function", + "<param", + "</function>", + "</param>", + "<think>", + "</think>", + }; + + data.thinking_start_tag = "<think>"; + data.thinking_end_tags = {"</think>"}; + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, + { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = "<|im_start|>assistant\n<think>\n" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "\n</think>\n\n" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal("<|im_start|>assistant\n"); + + auto reasoning = p.eps(); + if (extract_reasoning) { + reasoning = ("<think>" << p.reasoning(p.until("</think>")) << "</think>") + p.space(); + } + + // Response format parser + if (has_response_format) { + return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema)); + } + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + // CDATA lets a value carry characters that would otherwise close the tag (e.g. + // </param>); capture the inner text only, excluding the CDATA markers. + auto string_value = p.choice({ + p.literal("<![CDATA[") + p.ac(p.tool_arg_string_value(p.until("]]>")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("</param>")), + p.negate(p.literal("<![CDATA[")) + p.ac(p.tool_arg_string_value(p.until("</param>")) + p.tool_arg_close(p.literal("</param>")), "</param>") + }); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + const std::string name = function.at("name"); + + std::vector<common_peg_parser> arg_rules; + foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) { + auto value_parser = p.eps(); + if (prop.schema->may_be_string()) { + value_parser = string_value; + } else { + value_parser = p.tool_arg_json_value( + p.schema(p.json(), "tool-" + name + "-arg-" + prop.name + "-schema", doc, *prop.schema) + ) + p.tool_arg_close(p.literal("</param>")); + } + + arg_rules.push_back(p.tool_arg( + p.tool_arg_open(p.literal("<param name=\"") + p.tool_arg_name(p.literal(prop.name)) + p.literal("\">")) + + value_parser + )); + }); + + auto args = p.eps(); + if (!arg_rules.empty()) { + args = p.zero_or_more(p.choice(arg_rules) + p.space()); + } + + auto tool_parser = p.tool( + p.tool_open(p.literal("<function name=\"") + p.tool_name(p.literal(name)) + p.literal("\">")) + << p.tool_args(args) + << p.tool_close(p.literal("</function>"))); + + tool_choice |= p.rule("tool-" + name, tool_parser); + }); + + auto max_calls = inputs.parallel_tool_calls ? -1 : 1; + auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls)); + + auto content = p.content(p.until("<function")); + + return generation_prompt + reasoning + content + tool_calls + p.end(); + } + + return generation_prompt + reasoning + p.content(p.rest()) + p.end(); + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<function" }, + }; + } + + return data; +} diff --git a/common/parsers/minimax-m3.cpp b/common/parsers/minimax-m3.cpp new file mode 100644 index 000000000000..7ea9bfe5a083 --- /dev/null +++ b/common/parsers/minimax-m3.cpp @@ -0,0 +1,229 @@ +#include "parsers.h" + +common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_MINIMAX_M3; + data.supports_thinking = true; + data.thinking_start_tag = "<mm:think>"; + data.thinking_end_tags = {"</mm:think>"}; + + // M3 prefixes every tool tag with the namespace token "]<]minimax[>["; + // params use the parameter name as the tag (<file_path>...</file_path>). + const std::string NS = "]<]minimax[>["; + const std::string THINK_START = "<mm:think>"; + const std::string THINK_END = "</mm:think>"; + const std::string FC_START = NS + "<tool_call>"; + const std::string FC_END = NS + "</tool_call>"; + const std::string INVOKE_END = NS + "</invoke>"; + + data.preserved_tokens = { + NS, + "<tool_call>", + "</tool_call>", + THINK_START, + THINK_END, + }; + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai" }, + { COMMON_CHAT_ROLE_USER, "]~b]user" }, + { COMMON_CHAT_ROLE_TOOL, "]~b]tool" }, + { COMMON_CHAT_ROLE_SYSTEM, "]~b]developer" }, + { COMMON_CHAT_ROLE_SYSTEM, "]~b]system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + const std::string GEN_PROMPT = data.generation_prompt; + + using mm3 = common_chat_peg_minimax_m3_mapper; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START); + auto end = p.end(); + + auto reasoning = p.eps(); + if (extract_reasoning) { + auto block = inputs.enable_thinking + ? p.literal(THINK_START) + p.space() + + p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END) + : p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END); + + // A turn without reasoning is prefixed with a bare </mm:think>, written either by the + // generation prompt (thinking_mode = "disabled") or by the model itself. + reasoning = p.optional(p.choice({ block, p.literal(THINK_END) })); + } + + if (has_response_format) { + auto response_format = p.rule("response-format", + p.literal("```json") + p.space() + + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + + p.space() + p.literal("```")); + return generation_prompt + reasoning + response_format + end; + } + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return generation_prompt + reasoning + p.content(p.rest()) + end; + } + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto params = common_chat_tool_parameters(function); + auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(params)); + + // The template expands argument values recursively in XML (see the to_xml() macro) + std::function<common_peg_parser(const common_chat_schema &, const std::string &, const std::string &)> value_of; + std::function<common_peg_parser(const common_chat_schema_object &, const std::string &)> members_of; + + auto element_of = [&](const std::string & tag, const common_chat_schema & schema, const std::string & rule_name) { + const std::string close = NS + "</" + tag + ">"; + return p.rule(rule_name, + p.tool_arg( + p.tool_arg_open( + p.literal(NS + "<") + + p.tool_arg_name(p.literal(tag)) + + p.literal(">")) + + value_of(schema, rule_name, close))); + }; + + value_of = [&](const common_chat_schema & schema, + const std::string & rule_name, + const std::string & close) -> common_peg_parser { + auto close_tag = p.tool_arg_close(p.literal(close)); + + // A string accepts anything, so a union with a string alternative is a string + if (schema.may_be_string()) { + return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close); + } + + if (schema.kind() == common_chat_schema::KIND_ANY_OF) { + std::vector<common_peg_parser> choices; + + size_t index = 0; + for (const auto & alternative : static_cast<const common_chat_schema_any_of &>(schema).children) { + const std::string alt_name = rule_name + "-" + std::to_string(index++); + + // There is a risk that this breaks streaming deltas, but that's a risk we + // assume to provide tool arg streaming. + choices.push_back(value_of(*alternative, alt_name, close)); + } + + return p.choice(choices); + } + + if (schema.kind() == common_chat_schema::KIND_OBJECT) { + const auto & object = static_cast<const common_chat_schema_object &>(schema); + if (!object.properties.empty()) { + return p.tag(mm3::TOOL_ARG_OBJECT, members_of(object, rule_name)) + p.space() + close_tag; + } + } + + if (schema.kind() == common_chat_schema::KIND_ARRAY) { + const std::string item_close = NS + "</item>"; + auto item = p.rule(rule_name + "-item", + p.tag(mm3::TOOL_ARG_ITEM, + p.literal(NS + "<item>") + + value_of(*static_cast<const common_chat_schema_array &>(schema).items, rule_name + "-item", item_close))); + return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag; + } + + return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, schema)) + close_tag; + }; + + // Required properties in schema order, then any number of optional ones in any order. + members_of = [&](const common_chat_schema_object & object, const std::string & rule_prefix) -> common_peg_parser { + std::vector<common_peg_parser> required_elements; + std::vector<common_peg_parser> optional_elements; + for (const auto & prop : object.properties) { + auto element = element_of(prop.name, *prop.schema, rule_prefix + "-" + prop.name); + (prop.required ? required_elements : optional_elements).push_back(element); + } + + common_peg_parser members = p.eps(); + for (size_t i = 0; i < required_elements.size(); i++) { + if (i > 0) { + members = members + p.space(); + } + members = members + required_elements[i]; + } + + if (!optional_elements.empty()) { + common_peg_parser any_optional = p.choice(); + for (const auto & element : optional_elements) { + any_optional |= element; + } + members = members + p.repeat(p.space() + any_optional, 0, -1); + } + + return members; + }; + + common_peg_parser invoke_body = p.eps(); + if (doc->root->kind() == common_chat_schema::KIND_OBJECT) { + invoke_body = members_of(static_cast<const common_chat_schema_object &>(*doc->root), "tool-" + name + "-arg"); + } + + auto func_parser = p.tool( + p.tool_open(p.literal(NS + "<invoke name=\"") + + p.tool_name(p.literal(name)) + p.literal("\">")) + + p.space() + invoke_body + p.space() + + p.tool_close(p.literal(INVOKE_END))); + + tool_choice |= p.rule("tool-" + name, func_parser); + }); + + auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + + common_peg_parser tool_calls = p.eps(); + if (inputs.parallel_tool_calls) { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + + p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); + } else { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); + } + + if (!require_tools) { + tool_calls = p.optional(tool_calls); + } + + auto content_before_tools = p.content(p.until(FC_START)); + return generation_prompt + reasoning + content_before_tools + tool_calls + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START }, + }; + } + + return data; +} diff --git a/common/parsers/ministral3.cpp b/common/parsers/ministral3.cpp new file mode 100644 index 000000000000..8bcd204c62e8 --- /dev/null +++ b/common/parsers/ministral3.cpp @@ -0,0 +1,126 @@ +#include "parsers.h" + +common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + // Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja + auto adjusted_messages = json::array(); + for (const auto & msg : inputs.messages) { + auto role = msg.value("role", ""); + if (role != "system" && role != "assistant") { + // Only adjust system and assistant messages. Interestingly, the system message may contain thinking. + adjusted_messages.push_back(msg); + continue; + } + + auto content = json::array(); + + // If message contains `reasoning_content`, add it as a block of type `thinking` + if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { + content.push_back({ + { "type", "thinking" }, + { "thinking", msg.at("reasoning_content").get<std::string>() }, + }); + } + + // If message contains `content`, add it as a block of type `text` + if (msg.contains("content")) { + if (msg.at("content").is_string()) { + content.push_back({ + { "type", "text" }, + { "text", msg.at("content").get<std::string>() }, + }); + } else if (msg.at("content").is_array()) { + auto blocks = msg.at("content"); + content.insert(blocks); + } + } + + auto adjusted = msg; + adjusted["content"] = content; + adjusted.erase("reasoning_content"); + adjusted_messages.push_back(adjusted); + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = true; + + data.supports_thinking = true; + data.thinking_start_tag = "[THINK]"; + data.thinking_end_tags = {"[/THINK]"}; + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.preserved_tokens = { + "[THINK]", + "[/THINK]", + "[TOOL_CALLS]", + "[ARGS]", + }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = "[THINK]" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "[/THINK]" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.eps(); + auto reasoning = + extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps(); + + // Response format parser + if (has_response_format) { + // Ministral wants to emit json surrounded by code fences + return generation_prompt + (reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```"); + } + + // Tool call parser + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + const auto schema = common_chat_tool_parameters(function); + + tool_choice |= + p.rule("tool-" + name, p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") + + p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))); + }); + + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + auto max_calls = inputs.parallel_tool_calls ? -1 : 1; + auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls)); + + return generation_prompt + (reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls); + } + + // Content only parser + include_grammar = false; + return generation_prompt + (reasoning << p.content(p.rest())); + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]" } + }; + } + + return data; +} diff --git a/common/parsers/muse-glimmer.cpp b/common/parsers/muse-glimmer.cpp new file mode 100644 index 000000000000..d03bf2d58f73 --- /dev/null +++ b/common/parsers/muse-glimmer.cpp @@ -0,0 +1,138 @@ +#include "parsers.h" + +// An assistant turn is rendered as one or more messages, each +// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is +// <|eom|> (more messages follow) or <|eot|> (end of turn): +// - chain-of-thought: to=self, terminated by <|eom|> +// - final answer: to=user, terminated by <|eot|> +// The generation prompt is just "<|start|>assistant"; the model emits its own +// " to=...<|message|>". +common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = "<|start|>assistant"; + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + + data.preserved_tokens = { + "<|start|>", "<|message|>", "<|eom|>", "<|eot|>", + // ATEM tool-call markup emitted on " to=<tool>" turns. + "<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>", + "</atem:invoke>", "</atem:function_calls>", + }; + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, + { COMMON_CHAT_ROLE_USER, "<|start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, + { COMMON_CHAT_ROLE_TOOL, "<|start|>tool" }, + }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + // Constrained grammar whenever tools are offered. + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto start = p.rule("start", p.literal("<|start|>assistant")); + + if (!extract_reasoning && !include_grammar) { + return start + p.content(p.rest()); + } + + if (extract_reasoning) { + p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>")); + } else { + p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>")); + } + auto analysis = p.ref("analysis"); + + auto recipient = p.optional(p.literal(" to=user")); + auto final_msg = p.rule("final", recipient + p.literal("<|message|>") + + p.content(p.until_one_of({ "<|eot|>", "<|eom|>" }))); + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto string_value = p.ac( + p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")), + "</atem:parameter>"); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + const std::string name = function.at("name"); + + std::vector<common_peg_parser> arg_rules; + foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) { + auto value_parser = p.eps(); + if (prop.schema->may_be_string()) { + value_parser = string_value; + } else { + value_parser = p.tool_arg_json_value( + p.schema(p.json(), "tool-" + name + "-arg-" + prop.name + "-schema", doc, *prop.schema)) + + p.tool_arg_close(p.literal("</atem:parameter>")); + } + + arg_rules.push_back(p.tool_arg( + p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop.name)) + p.literal("\">")) + + value_parser)); + }); + + auto args = p.eps(); + if (!arg_rules.empty()) { + args = p.zero_or_more(p.choice(arg_rules) + p.space()); + } + + auto tool_parser = p.tool( + p.tool_open(p.literal(" to=") + p.until("<|message|>") + + p.literal("<|message|><atem:function_calls>") + p.space() + + p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space()) + << p.tool_args(args) + << p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>"))); + + tool_choice |= p.rule("tool-" + name, tool_parser); + }); + + auto tool_calls = inputs.parallel_tool_calls + ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice)) + : p.trigger_rule("tool-call", tool_choice); + + + if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { + return p.zero_or_more(start + analysis) + start + tool_calls; + } + auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls); + return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls)); + } + + return p.zero_or_more(start + analysis) + start + final_msg; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, + "(?:^|<\\|start\\|>assistant)( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" }, + }; + } + + return data; +} diff --git a/common/parsers/parsers.cpp b/common/parsers/parsers.cpp new file mode 100644 index 000000000000..643186c34cf2 --- /dev/null +++ b/common/parsers/parsers.cpp @@ -0,0 +1,25 @@ +#include "parsers.h" + +#include "log.h" + +void foreach_function(const json & tools, const std::function<void(const json &)> & fn) { + for (const auto & tool : tools) { + if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) { + LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str()); + continue; + } + fn(tool); + } +} + +void foreach_parameter(const json & function, const std::function<void(const common_chat_schema_property &, const common_chat_schema_document_ptr &)> & fn) { + auto params = common_chat_tool_parameters(function); + auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(params)); + const auto * object = dynamic_cast<const common_chat_schema_object *>(doc->root.get()); + if (!object) { + return; + } + for (const auto & prop : object->properties) { + fn(prop, doc); + } +} diff --git a/common/parsers/parsers.h b/common/parsers/parsers.h new file mode 100644 index 000000000000..f866360073bb --- /dev/null +++ b/common/parsers/parsers.h @@ -0,0 +1,79 @@ +#pragma once + +#include "chat.h" +#include "chat-auto-parser.h" +#include "chat-auto-parser-helpers.h" +#include "chat-peg-parser.h" +#include "common.h" +#include "ggml.h" +#include "json-schema-to-grammar.h" +#include "json.h" + +#include <functional> +#include <optional> +#include <set> +#include <string> +#include <vector> + +using json = common_json; + +// iterate over the function tools of an OpenAI-style tools array +void foreach_function(const json & tools, const std::function<void(const json &)> & fn); + +// iterate over the parameters of a function tool, with the document that owns them +void foreach_parameter(const json & function, const std::function<void(const common_chat_schema_property &, const common_chat_schema_document_ptr &)> & fn); + +// render a template; the override arguments let a parser feed in messages, tools or context it has rewritten +std::string common_chat_template_direct_apply_impl( + const common_chat_template & tmpl, + const autoparser::generation_params & inputs, + const std::optional<json> & messages_override = std::nullopt, + const std::optional<json> & tools_override = std::nullopt, + const std::optional<json> & additional_context = std::nullopt); + +// the suffix a template appends when add_generation_prompt is set +std::string common_chat_template_generation_prompt_impl( + const common_chat_template & tmpl, + const autoparser::generation_params & inputs, + const std::optional<json> & messages_override = std::nullopt, + const std::optional<json> & tools_override = std::nullopt, + const std::optional<json> & additional_context = std::nullopt); + +bool is_lfm2_template(const std::string & src); + +namespace workaround { + +void convert_tool_responses_gemma4(json & messages); + +} + +common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_gigachat_v3(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_ling3(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +// tool_list_tokens preserves the LFM2 system tool-list markers; LFM2.5 renders without them +common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl, const autoparser::generation_params & inputs, bool tool_list_tokens); + +common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + +common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, const autoparser::generation_params & inputs); diff --git a/common/parsers/qwen3-coder.cpp b/common/parsers/qwen3-coder.cpp new file mode 100644 index 000000000000..208f551f1b2b --- /dev/null +++ b/common/parsers/qwen3-coder.cpp @@ -0,0 +1,194 @@ +#include "parsers.h" + +common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + const std::string GEN_PREFIX = "<|im_start|>assistant\n"; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + + auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos; + + data.supports_thinking = supports_reasoning; + data.preserved_tokens = { + "<tool_call>", + "</tool_call>", + }; + + auto is_qwen3_coder = !supports_reasoning; + + if (supports_reasoning) { + data.thinking_start_tag = "<think>"; + // Support both </think> and <tool_call> as reasoning end sequences. + // The newline variant comes first so it is included in the forced message + // <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model + data.thinking_end_tags = { "\n</think>", "</think>", "<tool_call>" }; + data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" }); + } + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3 + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash + { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PREFIX; + if (supports_reasoning) { + data.generation_prompt += "<think>\n" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "\n</think>\n\n"; + } + } + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + std::vector<std::string> tool_call_starts = { "<tool_call>" }; + + if (is_qwen3_coder) { + // Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the + // starting <tool_call>. The model may hallucinate a tool name, but it is preferable over + // constraining on <function which may occur in valid content generation, e.g. #include <functional> + foreach_function(inputs.tools, [&](const json & tool) { + const std::string name = tool.at("function").at("name"); + tool_call_starts.push_back("<function=" + name + ">"); + }); + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PREFIX); + + auto reasoning = p.eps(); + if (supports_reasoning && extract_reasoning) { + reasoning = p.optional("<think>" + p.space() + + p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) + + (p.literal("</think>") | p.peek(p.literal("<tool_call>")))); + } + + // Response format parser + if (has_response_format) { + return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema))); + } + + // Tool call parser + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto arg_close = p.tool_arg_close(p.literal("\n</parameter>\n")); + auto arg_string = p.rule("xml-arg-string", + p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n")); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + + std::vector<common_peg_parser> required_args; + std::vector<common_peg_parser> optional_args; + + foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) { + auto rule_name = "tool-" + name + "-arg-" + param.name; + + auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param.name)) + ">\n"); + + auto types = param.schema->value_types(); + + auto arg_value = p.eps(); + if (!types.has(common_chat_schema::TYPE_STRING)) { + arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close; + } else if (types.is_only(common_chat_schema::TYPE_STRING)) { + arg_value = arg_string; + } else { + // The string alternative accepts any text, so the grammar only keeps the raw string + // rule. The parser still tries the JSON alternatives first to type the value. + auto json_value = p.choice(); + if (types.has(common_chat_schema::TYPE_OBJECT)) { + json_value |= p.json_object(); + } + if (types.has(common_chat_schema::TYPE_ARRAY)) { + json_value |= p.json_array(); + } + if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) { + json_value |= p.json_number(); + } + if (types.has(common_chat_schema::TYPE_BOOLEAN)) { + json_value |= p.json_bool(); + } + if (types.has(common_chat_schema::TYPE_NULL)) { + json_value |= p.json_null(); + } + arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "xml-arg-string"); + } + + auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); + + (param.required ? required_args : optional_args).push_back(arg_rule); + }); + + // Accept required arguments in any order, as Qwen does not always adhere to the + // order provided. + auto args = p.permute("tool-" + name + "-args", required_args); + if (!optional_args.empty()) { + args = args + p.zero_or_more(p.choice(optional_args)); + } + + auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") + + p.tool_args(args) + + p.tool_close(p.literal("</function>\n"))); + + tool_choice |= p.rule("tool-" + name, func); + }); + + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + + auto tool_call_body = tool_choice + "</tool_call>" + p.space(); + auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body); + + // Qwen3-Coder models may occasionally omit the <tool_call> token. + auto tool_call_first = is_qwen3_coder ? + p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body) : + tool_call; + + auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first; + auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1)); + + return generation_prompt + + (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls); + } + + // Content only parser + return generation_prompt + (reasoning << p.content(p.rest())); + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, data.grammar_lazy); + }); + + if (data.grammar_lazy) { + for (const auto & start : tool_call_starts) { + data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start }); + } + } + } + + return data; +} diff --git a/common/parsers/sources.cmake b/common/parsers/sources.cmake new file mode 100644 index 000000000000..70af84e25110 --- /dev/null +++ b/common/parsers/sources.cmake @@ -0,0 +1,21 @@ +# Specialized chat template parsers, listed explicitly so that adding or removing one re-runs CMake instead of leaving an incremental build stale. + +set(LLAMA_CHAT_PARSERS_SOURCES + ${CMAKE_CURRENT_LIST_DIR}/parsers.cpp + ${CMAKE_CURRENT_LIST_DIR}/parsers.h + ${CMAKE_CURRENT_LIST_DIR}/cohere2moe.cpp + ${CMAKE_CURRENT_LIST_DIR}/deepseek.cpp + ${CMAKE_CURRENT_LIST_DIR}/functionary-v3-2.cpp + ${CMAKE_CURRENT_LIST_DIR}/gemma4.cpp + ${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp + ${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp + ${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp + ${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp + ${CMAKE_CURRENT_LIST_DIR}/ling3.cpp + ${CMAKE_CURRENT_LIST_DIR}/lfm2.cpp + ${CMAKE_CURRENT_LIST_DIR}/minicpm5.cpp + ${CMAKE_CURRENT_LIST_DIR}/minimax-m3.cpp + ${CMAKE_CURRENT_LIST_DIR}/ministral3.cpp + ${CMAKE_CURRENT_LIST_DIR}/muse-glimmer.cpp + ${CMAKE_CURRENT_LIST_DIR}/qwen3-coder.cpp +) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 46fc29bf2f8b..75a908a281e1 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -166,6 +166,25 @@ common_peg_ast_id common_peg_ast_arena::find_by_rule(const common_peg_ast_node & return COMMON_PEG_INVALID_AST_ID; } +std::string common_peg_ast_node::sanitized_text() const { + if (invalid_utf8.empty()) { + return std::string(text); + } + + std::string out; + out.reserve(text.size() + 2 * invalid_utf8.size()); + + size_t seg_start = start; + for (const auto & invalid : invalid_utf8) { + out.append(text.data() + (seg_start - start), invalid.pos - seg_start); + out.append("\xEF\xBF\xBD"); + seg_start = invalid.pos + invalid.len; + } + out.append(text.data() + (seg_start - start), end - seg_start); + + return out; +} + void common_peg_ast_arena::visit(common_peg_ast_id id, const common_peg_ast_visitor & visitor) const { if (id == COMMON_PEG_INVALID_AST_ID) { return; @@ -282,6 +301,7 @@ struct parser_executor { auto pos = start_pos; std::vector<common_peg_ast_id> nodes; + std::vector<common_peg_invalid_utf8> invalid_utf8; for (size_t i = 0; i < p.children.size(); i++) { const auto & child_id = p.children[i]; @@ -306,13 +326,14 @@ struct parser_executor { if (!result.nodes.empty()) { nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end()); } + invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end()); if (result.need_more_input()) { ctx.parse_depth--; if (ctx.is_debug()) { fprintf(stderr, "%sSEQ -> NEED_MORE\n", debug_indent().c_str()); } - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes)); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes), std::move(invalid_utf8)); } pos = result.end; @@ -322,7 +343,7 @@ struct parser_executor { if (ctx.is_debug()) { fprintf(stderr, "%sSEQ -> SUCCESS at %zu->%zu\n", debug_indent().c_str(), start_pos, pos); } - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes)); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes), std::move(invalid_utf8)); } common_peg_parse_result operator()(const common_peg_choice_parser & p) { @@ -370,6 +391,7 @@ struct parser_executor { auto pos = start_pos; int match_count = 0; std::vector<common_peg_ast_id> nodes; + std::vector<common_peg_invalid_utf8> invalid_utf8; // Try to match up to max_count times (or unlimited if max_count is -1) while (p.max_count == -1 || match_count < p.max_count) { @@ -400,6 +422,7 @@ struct parser_executor { if (!result.nodes.empty()) { nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end()); } + invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end()); pos = result.end; match_count++; @@ -410,13 +433,14 @@ struct parser_executor { if (!result.nodes.empty()) { nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end()); } + invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end()); ctx.parse_depth--; if (ctx.is_debug()) { fprintf(stderr, "%sREPEAT -> NEED_MORE (count=%d, nodes=%zu)\n", debug_indent().c_str(), match_count, nodes.size()); } - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes)); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes), std::move(invalid_utf8)); } // Child failed - stop trying @@ -434,7 +458,7 @@ struct parser_executor { fprintf(stderr, "%sREPEAT -> NEED_MORE (not enough matches: %d < %d)\n", debug_indent().c_str(), match_count, p.min_count); } - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos, std::move(nodes)); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos, std::move(nodes), std::move(invalid_utf8)); } if (ctx.is_debug()) { fprintf(stderr, "%sREPEAT -> FAIL (not enough matches: %d < %d)\n", debug_indent().c_str(), match_count, @@ -448,7 +472,7 @@ struct parser_executor { fprintf(stderr, "%sREPEAT -> SUCCESS (count=%d, nodes=%zu)\n", debug_indent().c_str(), match_count, nodes.size()); } - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes)); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes), std::move(invalid_utf8)); } common_peg_parse_result operator()(const common_peg_and_parser & p) { @@ -664,23 +688,23 @@ struct parser_executor { // Scan input and check for delimiters size_t pos = start_pos; size_t last_valid_pos = start_pos; + std::vector<common_peg_invalid_utf8> invalid_utf8; while (pos < ctx.input.size()) { auto utf8_result = common_parse_utf8_codepoint(ctx.input, pos); - if (utf8_result.status == utf8_parse_result::INCOMPLETE) { - // Incomplete UTF-8 sequence - if (!ctx.is_lenient()) { - // Input is complete but UTF-8 is incomplete = malformed - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); - } - // Return what we have so far (before incomplete sequence) - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos); + if (utf8_result.status == utf8_parse_result::INCOMPLETE && ctx.is_lenient()) { + // The rest of the sequence may still arrive, return what we have so far + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8)); } - if (utf8_result.status == utf8_parse_result::INVALID) { - // Malformed UTF-8 - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); + if (utf8_result.status != utf8_parse_result::SUCCESS) { + // Malformed UTF-8, or a sequence truncated by the end of a complete input. + // A delimiter cannot start inside bytes that fail to decode, so consume them and move on + invalid_utf8.push_back({pos, utf8_result.bytes_consumed}); + pos += utf8_result.bytes_consumed; + last_valid_pos = pos; + continue; } // Check if a delimiter starts at this position @@ -688,12 +712,12 @@ struct parser_executor { if (match == common_trie::COMPLETE_MATCH) { // Found a complete delimiter, return everything before it - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8)); } if (match == common_trie::PARTIAL_MATCH) { // Found a partial match extending to end of input, return everything before it - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8)); } pos += utf8_result.bytes_consumed; @@ -702,9 +726,9 @@ struct parser_executor { if (last_valid_pos == ctx.input.size() && ctx.is_lenient()) { // Reached the end of a partial stream, there might still be more input that we need to consume. - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8)); } - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos, {}, std::move(invalid_utf8)); } common_peg_parse_result operator()(const common_peg_schema_parser & p) { @@ -728,10 +752,11 @@ struct parser_executor { result.end, text, std::move(result.nodes), - result.need_more_input() + result.need_more_input(), + result.invalid_utf8 ); - return common_peg_parse_result(result.type, result.start, result.end, { node_id }); + return common_peg_parse_result(result.type, result.start, result.end, { node_id }, std::move(result.invalid_utf8)); } return result; @@ -757,10 +782,11 @@ struct parser_executor { result.end, text, std::move(result.nodes), - result.need_more_input() + result.need_more_input(), + result.invalid_utf8 ); - return common_peg_parse_result(result.type, result.start, result.end, { node_id }); + return common_peg_parse_result(result.type, result.start, result.end, { node_id }, std::move(result.invalid_utf8)); } return result; @@ -953,7 +979,7 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id } else if constexpr (std::is_same_v<T, common_peg_until_parser>) { return "Until(" + string_join(p.delimiters, " | ") + ")"; } else if constexpr (std::is_same_v<T, common_peg_schema_parser>) { - return "Schema(" + dump_impl(p.child, visited) + ", " + (p.schema ? p.schema->dump() : "null") + ")"; + return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_chat_schema::kind_name(p.node->kind()) : "null") + ")"; } else if constexpr (std::is_same_v<T, common_peg_rule_parser>) { return "Rule(" + p.name + ", " + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v<T, common_peg_ref_parser>) { @@ -1119,8 +1145,13 @@ common_peg_parser common_peg_parser_builder::chars(const std::string & classes, return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max})); } +common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw) { + return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::move(doc), &node, raw})); +} + common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) { - return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared<common_json>(schema), raw})); + auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(schema)); + return this->schema(p, name, doc, *doc->root, raw); } common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) { @@ -1573,30 +1604,9 @@ static std::set<std::string> collect_reachable_rules( // GBNF generation implementation void common_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const { + // A raw string value is parsed by the child rather than constrained by the schema auto schema_delegates = [](const common_peg_schema_parser & s) -> bool { - if (!s.schema) { - return true; - } - if (s.raw && s.schema->contains("type")) { - const auto & type_val = s.schema->at("type"); - if (type_val.is_string() && type_val == "string") { - return true; - } - // Handle nullable types like ["string", "null"] - delegate when the - // non-null type is string, since the tagged format uses raw text - if (type_val.is_array()) { - for (const auto & t : type_val) { - if (t.is_string() && t.get<std::string>() != "null") { - return t.get<std::string>() == "string"; - } - } - } - } - // Delegate for enum schemas in raw mode - enum values are literal strings - if (s.raw && !s.schema->contains("type") && s.schema->contains("enum")) { - return true; - } - return false; + return !s.node || (s.raw && s.node->may_be_string()); }; // Unwrap the parser so we can properly check if it's a sequence or choice @@ -1731,7 +1741,7 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo if (schema_delegates(p)) { return to_gbnf(p.child); } - return builder.add_schema(p.name, *p.schema); + return builder.add_schema(p.name, *p.node); } else if constexpr (std::is_same_v<T, common_peg_rule_parser>) { return p.name; } else if constexpr (std::is_same_v<T, common_peg_ref_parser>) { @@ -1859,7 +1869,6 @@ static common_json serialize_parser_variant(const common_peg_parser_variant & va {"type", "schema"}, {"child", p.child}, {"name", p.name}, - {"schema", p.schema ? *p.schema : json(nullptr)}, {"raw", p.raw} }; } else if constexpr (std::is_same_v<T, common_peg_rule_parser>) { @@ -1999,15 +2008,12 @@ static common_peg_parser_variant deserialize_parser_variant(const common_json & return common_peg_until_parser{j["delimiters"].get<std::vector<std::string>>()}; } if (type == "schema") { - if (!j.contains("child") || !j.contains("name") || !j.contains("schema") || !j.contains("raw")) { + if (!j.contains("child") || !j.contains("name") || !j.contains("raw")) { throw std::runtime_error("schema parser missing required fields"); } common_peg_schema_parser parser; parser.child = j["child"].get<common_peg_parser_id>(); parser.name = j["name"]; - if (!j["schema"].is_null()) { - parser.schema = std::make_shared<common_json>(j["schema"]); - } parser.raw = j["raw"].get<bool>(); return parser; } diff --git a/common/peg-parser.h b/common/peg-parser.h index ab095cc7d671..8883259457ea 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -1,5 +1,6 @@ #pragma once +#include "json-schema.h" #include "json.h" #include <memory> @@ -71,6 +72,12 @@ enum common_peg_parse_result_type { const char * common_peg_parse_result_type_name(common_peg_parse_result_type type); +// A run of input bytes that does not decode as UTF-8 +struct common_peg_invalid_utf8 { + size_t pos; + size_t len; +}; + struct common_peg_ast_node { common_peg_ast_id id; std::string rule; @@ -81,6 +88,12 @@ struct common_peg_ast_node { std::vector<common_peg_ast_id> children; bool is_partial = false; + + // Invalid UTF-8 inside the node, in ascending order + std::vector<common_peg_invalid_utf8> invalid_utf8; + + // Returns the text with every invalid run replaced by U+FFFD + std::string sanitized_text() const; }; struct common_peg_parse_result; @@ -97,10 +110,11 @@ class common_peg_ast_arena { size_t end, std::string_view text, std::vector<common_peg_ast_id> children, - bool is_partial = false + bool is_partial = false, + std::vector<common_peg_invalid_utf8> invalid_utf8 = {} ) { common_peg_ast_id id = nodes_.size(); - nodes_.push_back({id, rule, tag, start, end, text, std::move(children), is_partial}); + nodes_.push_back({id, rule, tag, start, end, text, std::move(children), is_partial, std::move(invalid_utf8)}); return id; } @@ -126,6 +140,9 @@ struct common_peg_parse_result { std::vector<common_peg_ast_id> nodes; + // Invalid UTF-8 consumed by this result, carried up to the enclosing AST nodes + std::vector<common_peg_invalid_utf8> invalid_utf8; + common_peg_parse_result() = default; common_peg_parse_result(common_peg_parse_result_type type, size_t start) @@ -134,8 +151,8 @@ struct common_peg_parse_result { common_peg_parse_result(common_peg_parse_result_type type, size_t start, size_t end) : type(type), start(start), end(end) {} - common_peg_parse_result(common_peg_parse_result_type type, size_t start, size_t end, std::vector<common_peg_ast_id> nodes) - : type(type), start(start), end(end), nodes(std::move(nodes)) {} + common_peg_parse_result(common_peg_parse_result_type type, size_t start, size_t end, std::vector<common_peg_ast_id> nodes, std::vector<common_peg_invalid_utf8> invalid_utf8 = {}) + : type(type), start(start), end(end), nodes(std::move(nodes)), invalid_utf8(std::move(invalid_utf8)) {} bool fail() const { return type == COMMON_PEG_PARSE_RESULT_FAIL; } bool need_more_input() const { return type == COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT; } @@ -245,7 +262,8 @@ struct common_peg_until_parser { struct common_peg_schema_parser { common_peg_parser_id child; std::string name; - std::shared_ptr<common_json> schema; + common_chat_schema_document_ptr doc; // owns node + const common_chat_schema * node = nullptr; // Indicates if the GBNF should accept a raw string that matches the schema. bool raw; @@ -428,6 +446,7 @@ class common_peg_parser_builder { common_peg_parser space() { return add(common_peg_space_parser{}); } // Matches all characters until a delimiter is found (delimiter not consumed). + // Invalid UTF-8 is consumed and recorded on the AST nodes. // S -> (!delim .)* common_peg_parser until(const std::string & delimiter) { return add(common_peg_until_parser{{delimiter}}); } @@ -488,8 +507,10 @@ class common_peg_parser_builder { // A marker, i.e. text delimited by a pair of <> or [] common_peg_parser marker(); - // Wraps a parser with JSON schema metadata for grammar generation. - // Used internally to convert JSON schemas to GBNF grammar rules. + // Wraps a parser with the schema its GBNF is generated from, a node of the document that owns it + common_peg_parser schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw = false); + + // Parses the JSON schema into a document of its own common_peg_parser schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw = false); // Creates a named rule, stores it in the grammar, and returns a ref. diff --git a/common/speculative.cpp b/common/speculative.cpp index e70d4ac7f7ad..fe4e55c5fcab 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -476,7 +476,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { drafting[seq_id] = true; common_sampler_reset(smpls[seq_id].get()); - common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); + common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true); } int ret = llama_decode(ctx_dft, batch); @@ -535,7 +535,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { continue; } - common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); + common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true); } if (batch.n_tokens == 0) { @@ -1617,8 +1617,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { // Target prefill may contain token IDs or multimodal embeddings. Both // produce the target-layer features used to seed the draft KV cache, so - // skipping the embedding batches leaves a hole in the draft's cache and - // the next injection fails to initialize. + // embeddings are injected too, except the pinned ones skipped below. // TODO: revisit after https://github.com/ggml-org/llama.cpp/pull/24669 is merged const bool has_tokens = batch_in.token != nullptr; const bool has_embeddings = batch_in.embd != nullptr; @@ -1666,6 +1665,13 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { } const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1; + // an M-RoPE image pins all its rows to one position, so a windowed draft + // cache cannot free cells for it - skip it, the draft can jump over the gap + const bool pos_pinned = batch_in.pos[i_batch_beg[seq_id]] == batch_in.pos[i_batch_end[seq_id]]; + if (has_embeddings && n_rows > 1 && pos_pinned) { + continue; + } + if (wp_spec_prefill_stats_enabled()) { SPC_INF("DFlash inject seq=%d input_rows=%d kept_rows=%d chunks=%d\n", seq_id, n_rows_in, n_rows, 1 + (n_rows - 1) / n_ubatch); @@ -1831,7 +1837,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { common_sampler_reset(smpls[seq_id].get()); - const int32_t n = (int32_t) dp.n_past; + const int32_t n = (int32_t) dp.pos0; const int32_t n_draft = params.n_max; @@ -2131,7 +2137,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { // conditioned on. If call N+1's proposals repeat call N's at the same // absolute positions while THIS line changes, the anchor is being ignored. SPC_INF("DBG anchor seq=%d call=%d n_past=%d id_last=%d '%s'\n", - seq_id, dbg_n_draft, (int) dp.n_past, dp.id_last, + seq_id, dbg_n_draft, (int) dp.pos0, dp.id_last, common_token_to_piece(ctx_dft, dp.id_last).c_str()); const float * conf_dbg = services_mode @@ -2176,7 +2182,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { SPC_INF("DBG slot seq=%d call=%d i=%d pos=%d conf=%.3e gap=%.3f | " "top1=%6d (%8.3f) '%s' | top2=%6d (%8.3f) '%s' | top3=%6d (%8.3f) '%s'%s\n", - seq_id, dbg_n_draft, i, (int) dp.n_past + i, c, v[0] - v[1], + seq_id, dbg_n_draft, i, (int) dp.pos0 + i, c, v[0] - v[1], t[0], v[0], common_token_to_piece(ctx_dft, t[0]).c_str(), t[1], v[1], common_token_to_piece(ctx_dft, t[1]).c_str(), t[2], v[2], common_token_to_piece(ctx_dft, t[2]).c_str(), @@ -2706,7 +2712,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } } - // remember the frist and last batch index for each sequence + // remember the first and last batch index for each sequence std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1); std::fill(i_batch_end.begin(), i_batch_end.end(), -1); @@ -3203,7 +3209,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { drafting[seq_id] = true; common_sampler_reset(smpls[seq_id].get()); - common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); + common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[seq_id].data(), row_bytes); i_last[seq_id] = batch.n_tokens - 1; @@ -3243,16 +3249,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { while (n_drafting > 0) { // each step decodes under a different head, i.e. a different decoder layer, and - // KV is per layer. process() filled this layer's KV only for positions < n_past + // KV is per layer. process() filled this layer's KV only for positions < pos0 // (prompt + accepted prefix) — nothing in the draft region yet. so reset the - // draft region (the seq_rm lower bound is n_past, leaving the prompt KV intact) + // draft region (the seq_rm lower bound is pos0, leaving the prompt KV intact) // and select head i so it rebuilds its own layer's KV there; decoding just the // latest token would leave its attention reading cells only another head wrote. if (chain_heads) { auto * mem_dft = llama_get_memory(ctx_dft); for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { if (drafting[seq_id]) { - llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].n_past, -1); + llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].pos0, -1); } } llama_set_nextn_layer_offset(ctx_dft, i); @@ -3347,17 +3353,17 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const int n_rows = (int) result.size() + 1; // id_last + tokens drafted so far for (int t = 0; t < n_rows; ++t) { const llama_token tok = (t == 0) ? dp.id_last : result[t - 1]; - common_batch_add(batch, tok, dp.n_past + t, { seq_id }, t == n_rows - 1); + common_batch_add(batch, tok, dp.pos0 + t, { seq_id }, t == n_rows - 1); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, chain_h[seq_id].data() + (size_t) t * n_embd, row_bytes); } } else if (is_mem_shared) { // note: with shared memory (e.g. Gemma4 assistants) we use the same position for all draft tokens // ref: https://github.com/huggingface/transformers/blob/effde20942e3f82a1b97449f60b3a48c5ff96145/docs/source/en/model_doc/gemma4_assistant.md?plain=1#L36-L37 - common_batch_add(batch, id, dp.n_past, { seq_id }, true); + common_batch_add(batch, id, dp.pos0, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } else { - common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); + common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } @@ -4200,7 +4206,10 @@ common_params common_base_params_to_speculative(const common_params & params) { result.pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED; if (has_draft) { - result.devices = params_spec.devices; + // default to global devices value + if (!params_spec.devices.empty()) { + result.devices = params_spec.devices; + } result.model = params_spec.mparams; // MAD-LAB: sidecars belong to the target GGUF; a separate draft model // must not pick up the target's Engram tables. diff --git a/common/speculative.h b/common/speculative.h index c122c7fcc99f..b69b9a9c736c 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -61,7 +61,7 @@ struct common_speculative_draft_params { // can be used to constraint the max draft based on the remaining context size int32_t n_max = -1; - llama_pos n_past; + llama_pos pos0; llama_token id_last; // TODO: remove in the future by keeping track of the prompt from the _begin() call and the consecutive accept calls diff --git a/common/unicode.cpp b/common/unicode.cpp index f71fe56783ff..4ebe94a9ec8c 100644 --- a/common/unicode.cpp +++ b/common/unicode.cpp @@ -26,16 +26,16 @@ utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t off // Invalid: continuation byte as first byte if (!(input[offset] & 0x40)) { - return utf8_parse_result(utf8_parse_result::INVALID); + return utf8_parse_result(utf8_parse_result::INVALID, 0, 1); } // 2-byte sequence if (!(input[offset] & 0x20)) { if (offset + 1 >= input.size()) { - return utf8_parse_result(utf8_parse_result::INCOMPLETE); + return utf8_parse_result(utf8_parse_result::INCOMPLETE, 0, 1); } if ((input[offset + 1] & 0xc0) != 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); + return utf8_parse_result(utf8_parse_result::INVALID, 0, 1); } auto result = ((input[offset] & 0x1f) << 6) | (input[offset + 1] & 0x3f); return utf8_parse_result(utf8_parse_result::SUCCESS, result, 2); @@ -43,11 +43,14 @@ utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t off // 3-byte sequence if (!(input[offset] & 0x10)) { - if (offset + 2 >= input.size()) { - return utf8_parse_result(utf8_parse_result::INCOMPLETE); - } - if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); + // Check one byte at a time so a bad byte is reported before a short input + for (size_t i = 1; i < 3; i++) { + if (offset + i >= input.size()) { + return utf8_parse_result(utf8_parse_result::INCOMPLETE, 0, i); + } + if ((input[offset + i] & 0xc0) != 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID, 0, i); + } } auto result = ((input[offset] & 0x0f) << 12) | ((input[offset + 1] & 0x3f) << 6) | (input[offset + 2] & 0x3f); return utf8_parse_result(utf8_parse_result::SUCCESS, result, 3); @@ -55,18 +58,20 @@ utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t off // 4-byte sequence if (!(input[offset] & 0x08)) { - if (offset + 3 >= input.size()) { - return utf8_parse_result(utf8_parse_result::INCOMPLETE); - } - if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80 || (input[offset + 3] & 0xc0) != 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); + for (size_t i = 1; i < 4; i++) { + if (offset + i >= input.size()) { + return utf8_parse_result(utf8_parse_result::INCOMPLETE, 0, i); + } + if ((input[offset + i] & 0xc0) != 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID, 0, i); + } } auto result = ((input[offset] & 0x07) << 18) | ((input[offset + 1] & 0x3f) << 12) | ((input[offset + 2] & 0x3f) << 6) | (input[offset + 3] & 0x3f); return utf8_parse_result(utf8_parse_result::SUCCESS, result, 4); } // Invalid first byte - return utf8_parse_result(utf8_parse_result::INVALID); + return utf8_parse_result(utf8_parse_result::INVALID, 0, 1); } bool common_utf8_is_complete(const std::string & s) { diff --git a/common/unicode.h b/common/unicode.h index 9b32fa19d62b..380e77e4c759 100644 --- a/common/unicode.h +++ b/common/unicode.h @@ -9,7 +9,7 @@ struct utf8_parse_result { uint32_t codepoint; // Decoded codepoint (only valid if status == SUCCESS) - size_t bytes_consumed; // How many bytes this codepoint uses (1-4) + size_t bytes_consumed; // How many bytes this codepoint uses (1-4), or the length of the valid prefix if status != SUCCESS enum status { SUCCESS, INCOMPLETE, INVALID } status; utf8_parse_result(enum status s, uint32_t cp = 0, size_t bytes = 0) diff --git a/conversion/__init__.py b/conversion/__init__.py index f9b9d883a391..18dbd13fbfd9 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -95,6 +95,7 @@ "Gemma3nForCausalLM": "gemma", "Gemma3nForConditionalGeneration": "gemma", "Gemma4AssistantForCausalLM": "gemma", + "Gemma4DSparkModel": "gemma", "Gemma4ForConditionalGeneration": "gemma", "Gemma4ForCausalLM": "gemma", "Gemma4UnifiedForConditionalGeneration": "gemma", @@ -124,6 +125,7 @@ "HunYuanDenseV1ForCausalLM": "hunyuan", "HunYuanMoEV1ForCausalLM": "hunyuan", "HunYuanVLForConditionalGeneration": "hunyuan", + "HrmTextForCausalLM": "hrm_text", "HYV3ForCausalLM": "hunyuan", "HYV4ForCausalLM": "hy_v4", "IQuestCoderForCausalLM": "llama", @@ -169,6 +171,7 @@ "Mamba2ForCausalLM": "mamba", "MambaForCausalLM": "mamba", "MambaLMHeadModel": "mamba", + "MapleForCausalLM": "maple", "MellumForCausalLM": "mellum", "MiMoV2FlashForCausalLM": "mimo", "MiMoV2ForCausalLM": "mimo", @@ -256,6 +259,7 @@ "SeedOssForCausalLM": "olmo", "SmallThinkerForCausalLM": "smallthinker", "SmolLM3ForCausalLM": "llama", + "Spark2_5ForCausalLM": "spark2_5", "SolarOpenForCausalLM": "glm", "StableLMEpochForCausalLM": "stablelm", "StableLmForCausalLM": "stablelm", diff --git a/conversion/base.py b/conversion/base.py index 1b14169f5f23..70ab92aae8e9 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -130,7 +130,8 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, sentence_transformers_dense_modules: bool = False, target_model_dir: Path | None = None, fuse_gate_up_exps: bool = False, - fp8_as_q8: bool = False): + fp8_as_q8: bool = False, + fuse_qkv: bool = False): if type(self) is ModelBase or \ type(self) is TextModel or \ type(self) is MmprojModel: @@ -153,6 +154,15 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, self.fuse_gate_up_exps = fuse_gate_up_exps self._gate_exp_buffer: dict[int, Tensor] = {} self._up_exp_buffer: dict[int, Tensor] = {} + self.fuse_qkv = fuse_qkv + self._q_buffer: dict[int, Tensor] = {} + self._k_buffer: dict[int, Tensor] = {} + self._v_buffer: dict[int, Tensor] = {} + self._q_bias_buffer: dict[int, Tensor] = {} + self._k_bias_buffer: dict[int, Tensor] = {} + self._v_bias_buffer: dict[int, Tensor] = {} + self._fusable_qkv_weight_layers: set[int] = set() + self._fusable_qkv_bias_layers: set[int] = set() self.hparams = ModelBase.load_hparams(self.dir_model, self.is_mistral_format) if hparams is None else hparams self.model_tensors = self.index_tensors(remote_hf_model_id=remote_hf_model_id) self.metadata_override = metadata_override @@ -621,6 +631,43 @@ def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", " raise ValueError(f"Can not map tensor {name!r}") return new_name + def prepare_qkv_fusion(self) -> None: + self._fusable_qkv_weight_layers.clear() + self._fusable_qkv_bias_layers.clear() + if not self.fuse_qkv or gguf.MODEL_TENSOR.ATTN_QKV not in gguf.MODEL_TENSORS[self.model_arch]: + return + + qkv_types = { + gguf.MODEL_TENSOR.ATTN_Q, + gguf.MODEL_TENSOR.ATTN_K, + gguf.MODEL_TENSOR.ATTN_V, + } + weights: dict[int, set[gguf.MODEL_TENSOR]] = {} + biases: dict[int, set[gguf.MODEL_TENSOR]] = {} + + for name in self.model_tensors: + mapped = self.tensor_map.get_type_and_name(name, try_suffixes=(".weight", ".bias")) + if mapped is None: + continue + tensor_type, new_name = mapped + if tensor_type not in qkv_types: + continue + + bid = next((int(part) for part in new_name.split(".") if part.isdecimal()), None) + if bid is None: + continue + if new_name.endswith(".weight"): + weights.setdefault(bid, set()).add(tensor_type) + elif new_name.endswith(".bias"): + biases.setdefault(bid, set()).add(tensor_type) + + for bid, weight_types in weights.items(): + bias_types = biases.get(bid, set()) + if weight_types == qkv_types and (not bias_types or bias_types == qkv_types): + self._fusable_qkv_weight_layers.add(bid) + if bias_types: + self._fusable_qkv_bias_layers.add(bid) + def set_gguf_parameters(self): raise NotImplementedError("set_gguf_parameters() must be implemented in subclasses") @@ -649,6 +696,40 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_UP_EXP, bid): return [] + # Handle Q/K/V tensor fusion if enabled + qkv_bid = next((int(part) for part in new_name.split(".") if part.isdecimal()), None) if self.fuse_qkv else None + if qkv_bid is not None: + is_bias = new_name.endswith('.bias') + suffix = '.bias' if is_bias else '.weight' + fusable_layers = self._fusable_qkv_bias_layers if is_bias else self._fusable_qkv_weight_layers + if qkv_bid not in fusable_layers: + return [(new_name, data_torch)] + + buf_q = self._q_bias_buffer if is_bias else self._q_buffer + buf_k = self._k_bias_buffer if is_bias else self._k_buffer + buf_v = self._v_bias_buffer if is_bias else self._v_buffer + + if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_Q, qkv_bid, suffix): + buf_q[qkv_bid] = data_torch + elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_K, qkv_bid, suffix): + buf_k[qkv_bid] = data_torch + elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_V, qkv_bid, suffix): + buf_v[qkv_bid] = data_torch + + if qkv_bid in buf_q and qkv_bid in buf_k and qkv_bid in buf_v: + q_data = buf_q.pop(qkv_bid) + k_data = buf_k.pop(qkv_bid) + v_data = buf_v.pop(qkv_bid) + fused_data = torch.cat([q_data, k_data, v_data], dim=0) + fused_name = self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, qkv_bid, suffix=suffix) + logger.info(f"Fused Q, K, V {suffix[1:]} into QKV for layer {qkv_bid}") + return [(fused_name, fused_data)] + + if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_Q, qkv_bid, suffix) or \ + self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_K, qkv_bid, suffix) or \ + self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_V, qkv_bid, suffix): + return [] + return [(new_name, data_torch)] def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: @@ -699,6 +780,36 @@ def repack_mxfp4_blocks(packed: Tensor, scale: Tensor) -> np.ndarray: raw = torch.cat((s.unsqueeze(-1), qs.to(torch.uint8)), dim=-1) return raw.reshape(rows, n_blocks * 17).cpu().numpy() + def _mxfp4_expert_tensor(self, loaders: list[tuple[Callable[[], Tensor], Callable[[], Tensor]]]): + """ + One stacked [n_expert, rows, cols] MXFP4 tensor, built lazily. + + gguf_writer holds every added tensor until the final write, so building + this eagerly (like the DeepSeek-V4 path does) keeps every expert in + memory at once. lazy means only the tensor being written is resident. + """ + # meta shapes, so this does not read any weights + rows, packed_cols = loaders[0][0]().shape + n_blocks = (packed_cols * 2) // 32 + byte_shape = (len(loaders), rows, n_blocks * 17) + + def load(fns: list[tuple[Callable[[], Tensor], Callable[[], Tensor]]]) -> np.ndarray: + out = np.empty(byte_shape, dtype=np.uint8) + for eid, (packed_fn, scale_fn) in enumerate(fns): + out[eid] = self.repack_mxfp4_blocks( + LazyTorchTensor.to_eager(packed_fn()), + LazyTorchTensor.to_eager(scale_fn()), + ) + return out + + # loaders goes through args, not the closure, so that `func` matches + # LazyBase's single-argument shape + return gguf.LazyNumpyTensor( + meta=gguf.LazyNumpyTensor.meta_with_dtype_and_shape(np.uint8, byte_shape), + args=(loaders,), + func=load, + ) + @staticmethod def _nvfp4_pack(weight: Tensor, scale: Tensor) -> tuple[np.ndarray, list[int]]: """Repack NVFP4 ModelOpt tensors into ggml super-block layout. @@ -903,6 +1014,8 @@ def load(): self.dequant_model() + self.prepare_qkv_fusion() + # Handle empty tensor_map for models with block_count=0 (like MobileNetV5) if self.tensor_map.mapping: max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,") @@ -1031,6 +1144,13 @@ def load(): self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype) + qkv_buffers = ( + self._q_buffer, self._k_buffer, self._v_buffer, + self._q_bias_buffer, self._k_bias_buffer, self._v_bias_buffer, + ) + if any(qkv_buffers): + raise ValueError("QKV fusion did not consume all buffered tensors") + def set_type(self): self.gguf_writer.add_type(gguf.GGUFType.MODEL) @@ -1551,6 +1671,12 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7": # ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B res = "lfm2" + if chkhsh == "846deafc5b0fa786186fa4ae6c7b49903cf2f1d1895bdb80b9120d60be135252": + # ref: https://huggingface.co/danish-foundation-models/DFM-Mimir + res = "gemma4" + if chkhsh == "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed": + # ref: https://huggingface.co/XHToken/Spark-X2.5-1.7B + res = "spark2_5" if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5": # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B res = "llama-bpe" @@ -1773,6 +1899,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "972da7b59cec44d1f0a490a86c96df53859e486e481563e5dddac155013d87ac": # ref: https://huggingface.co/poolside/Laguna-XS.2 res = "laguna" + if chkhsh == "653660222fb704f61cbf2b618a8ae6502b7f8b20c980f9a5de07ed78e13319cd": + # ref: https://huggingface.co/ufakai/ufakzeka-1 + res = "ufakzeka" if res is None: logger.warning("\n") diff --git a/conversion/gemma.py b/conversion/gemma.py index 6b4d7d17154d..9ec622ed4995 100644 --- a/conversion/gemma.py +++ b/conversion/gemma.py @@ -11,6 +11,7 @@ from torch import Tensor from .base import MmprojModel, ModelBase, TextModel, gguf, logger +from .qwen import DFlashModel @ModelBase.register("GemmaForCausalLM") @@ -809,6 +810,105 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter yield from super().modify_tensors(data_torch, name, bid) +@ModelBase.register("Gemma4DSparkModel") +class Gemma4DSparkModel(DFlashModel): + model_arch = gguf.MODEL_ARCH.DFLASH + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + if not self.hparams.get("attention_k_eq_v", False): + raise ValueError("Gemma4 DSpark currently requires attention_k_eq_v") + if self.hparams.get("layer_types") != ["full_attention"] * self.block_count: + raise ValueError("Gemma4 DSpark currently requires uniform full_attention layer types") + if self.hparams.get("hidden_activation", "gelu_pytorch_tanh") != "gelu_pytorch_tanh": + raise ValueError("Gemma4 DSpark currently requires hidden_activation=gelu_pytorch_tanh") + if self.hparams.get("attention_bias", False) or self.hparams.get("enable_moe_block", False): + raise ValueError("Gemma4 DSpark attention bias and MoE are not supported") + if (self.hparams.get("draft_vocab_size") or self.hparams["vocab_size"]) != self.hparams["vocab_size"]: + raise ValueError("Gemma4 DSpark currently requires a full draft vocabulary") + if "model.lm_head.weight" not in self.model_tensors and self.hparams.get("tie_word_embeddings") is not True: + raise ValueError("Gemma4 DSpark requires lm_head.weight unless tie_word_embeddings is true") + + self.dflash_config = self.hparams.get("dflash_config", {}) + markov_type = self.dflash_config.get("markov_head_type", self.hparams.get("markov_head_type", "vanilla")) + if markov_type != "vanilla": + raise ValueError("Gemma4 DSpark currently requires a vanilla Markov head") + + # Gemma4TextConfig supplies these defaults when rope_parameters is absent. + rope = self.hparams.get("rope_parameters") or { + "full_attention": {"rope_type": "proportional", "partial_rotary_factor": 0.25, "rope_theta": 1000000.0}, + } + self.rope_parameters = rope.get("full_attention", rope) + if self.rope_parameters.get("rope_type") not in ("default", "proportional"): + raise ValueError("Gemma4 DSpark requires default or proportional RoPE") + + def set_vocab(self): + super().set_vocab() + mask_id = self.dflash_config.get("mask_token_id", self.hparams.get("mask_token_id")) + if mask_id is None: + raise ValueError("Gemma4 DSpark requires mask_token_id") + if "mask_token_id" not in self.dflash_config: + self.gguf_writer.add_mask_token_id(mask_id) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + head_dim = int(self.hparams["global_head_dim"]) + self.gguf_writer.add_head_count_kv(self.hparams["num_global_key_value_heads"]) + self.gguf_writer.add_key_length(head_dim) + self.gguf_writer.add_value_length(head_dim) + self.gguf_writer.add_rope_dimension_count(head_dim) + self.gguf_writer.add_embedding_scale(self.hparams["hidden_size"] ** 0.5) + self.gguf_writer.add_attention_scale(1.0) + self.gguf_writer.add_hidden_act("gelu_pytorch_tanh") + + self.gguf_writer.add_sample_from_anchor(self.hparams.get("sample_from_anchor", True)) + target_layers = self.dflash_config.get("target_layer_ids", self.hparams.get("target_layer_ids")) + if not target_layers: + raise ValueError("Gemma4 DSpark requires target_layer_ids") + self.gguf_writer.add_has_confidence_head(any("confidence_head.proj" in name for name in self.model_tensors)) + + if self.hparams.get("final_logit_softcapping"): + raise ValueError("Gemma4 DSpark logit softcapping is not supported") + # The top-level sliding_window is inert unless the draft enables SWA. + if self.dflash_config.get("use_swa", False): + window = self.dflash_config["swa_window_size"] + if window <= 0: + raise ValueError("Gemma4 DSpark swa_window_size must be positive") + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if not name.startswith("model."): + name = "model." + name + if name.endswith(".layer_scalar"): + name += ".weight" + name = name.replace("model.confidence_proj.", "model.confidence_head.proj.") + return super().filter_tensors((name, gen)) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # The shared DFlash map assigns this name to Qwen's pre-FFN norm. + if name.endswith(".post_attention_layernorm.weight"): + name = self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_POST_NORM, bid) + elif name.endswith(".pre_feedforward_layernorm.weight"): + name = self.format_tensor_name(gguf.MODEL_TENSOR.FFN_NORM, bid) + yield from super().modify_tensors(data_torch, name, bid) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + if self.rope_parameters["rope_type"] == "proportional": + # Keep the unrotated dimensions in place, as in the Gemma4 converter. + head_dim = int(self.hparams["global_head_dim"]) + fraction_value = self.rope_parameters.get("partial_rotary_factor", 0.25) + if not isinstance(fraction_value, (int, float)): + raise ValueError("Gemma4 DSpark partial_rotary_factor must be numeric") + fraction = float(fraction_value) + n_rot = int(head_dim * fraction / 2) + if not 0 < fraction <= 1 or head_dim * fraction != 2 * n_rot: + raise ValueError("Gemma4 DSpark rotary dimension count must be positive and even") + factors = torch.tensor([1.0] * n_rot + [1e30] * (head_dim // 2 - n_rot), dtype=torch.float32) + yield self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), factors + + @ModelBase.register("Gemma4UnifiedForConditionalGeneration") @ModelBase.example("hf-tiny-v2/tiny-random-Gemma4UnifiedForConditionalGeneration") class Gemma4UnifiedModel(Gemma4Model): diff --git a/conversion/hrm_text.py b/conversion/hrm_text.py new file mode 100644 index 000000000000..3684fe912a15 --- /dev/null +++ b/conversion/hrm_text.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import re + +from typing import Iterable, TYPE_CHECKING + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, gguf + + +@ModelBase.register("HrmTextForCausalLM") +@ModelBase.example("danish-foundation-models/DFM-Mimir") +class HrmTextModel(TextModel): + model_arch = gguf.MODEL_ARCH.HRM_TEXT + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # training-style configs store the per-stack count in num_hidden_layers, + # transformers-style configs keep it in num_layers_per_stack + self.layers_per_stack = self.hparams.get("num_layers_per_stack") or self.hparams["num_hidden_layers"] + self.h_cycles = self.hparams["H_cycles"] + self.l_cycles = self.hparams["L_cycles"] + + # block_count is the expanded cache-slot count; the file only holds + # 2 * layers_per_stack physical blocks + self.block_count = self.layers_per_stack * self.h_cycles * (self.l_cycles + 1) + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, 2 * self.layers_per_stack) + + def set_vocab(self): + self._set_vocab_gpt2() + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + head_dim = self.hparams.get("head_dim") or self.hparams["hidden_size"] // self.hparams["num_attention_heads"] + self.gguf_writer.add_rope_dimension_count(head_dim) + self.gguf_writer.add_embedding_scale(self.hparams["embedding_scale"]) + self.gguf_writer.add_hrm_layers_per_stack(self.layers_per_stack) + self.gguf_writer.add_hrm_h_cycles(self.h_cycles) + self.gguf_writer.add_hrm_l_cycles(self.l_cycles) + self.gguf_writer.add_hrm_prefix_lm(bool(self.hparams.get("prefix_lm", False))) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name == "model.embed_tokens.weight": + yield self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), data_torch + return + if name == "lm_head.weight": + yield self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch + return + if name == "model.z_L_init": + yield self.format_tensor_name(gguf.MODEL_TENSOR.HRM_Z_L_INIT, suffix=""), data_torch + return + + match = re.fullmatch(r"model\.([LH])_module\.layers\.(\d+)\.(.+)", name) + if match is None: + raise ValueError(f"can not map tensor: {name}") + + stack, layer_s, tensor_name = match.groups() + # the L stack occupies blocks [0, layers_per_stack), the H stack follows it + layer_idx = int(layer_s) + (self.layers_per_stack if stack == "H" else 0) + + if tensor_name == "attn.gqkv_proj.weight": + gate, q, k, v = data_torch.chunk(4, dim=0) + yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_GATE, layer_idx), gate.contiguous() + yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, layer_idx), q.contiguous() + yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, layer_idx), k.contiguous() + yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, layer_idx), v.contiguous() + elif tensor_name == "mlp.gate_up_proj.weight": + gate, up = data_torch.chunk(2, dim=0) + yield self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, layer_idx), gate.contiguous() + yield self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, layer_idx), up.contiguous() + else: + if tensor_name.startswith("attn."): + tensor_name = "self_attn." + tensor_name[len("attn."):] + tensor_name = "model.layers.{bid}." + tensor_name + yield from super().modify_tensors(data_torch, tensor_name.format(bid=layer_idx), layer_idx) diff --git a/conversion/hunyuan.py b/conversion/hunyuan.py index ee1a10654523..58dee2ec8bc2 100644 --- a/conversion/hunyuan.py +++ b/conversion/hunyuan.py @@ -159,32 +159,14 @@ def prepare_tensors(self): class HunYuanModel(TextModel): model_arch = gguf.MODEL_ARCH.HUNYUAN_DENSE - def _get_eod_token_id(self) -> int | None: - """Get the actual end-of-generation token from config (eod_token_id).""" - return self.hparams.get("eod_token_id") - - def _get_eot_token_id(self) -> int | None: - """Get the end-of-turn token from generation_config.json. - This is the first entry in eos_token_id when it's a list.""" - gen_cfg_path = self.dir_model / "generation_config.json" - if gen_cfg_path.is_file(): - with open(gen_cfg_path, encoding="utf-8") as f: - gen_cfg = json.load(f) - eos = gen_cfg.get("eos_token_id") - if isinstance(eos, list) and len(eos) >= 2: - return eos[0] - return None - - def _fix_special_tokens(self): - """Fix EOS/EOT tokens that are incorrect in upstream configs.""" - eod_id = self._get_eod_token_id() - if eod_id is not None: - self.gguf_writer.add_eos_token_id(eod_id) - eot_id = self._get_eot_token_id() - if eot_id is not None: - self.gguf_writer.add_eot_token_id(eot_id) - def set_vocab(self): + # Also called by draft models (e.g. DFlash), with dir_model pointing at + # the target model. + config = ModelBase.load_hparams(self.dir_model, self.is_mistral_format) + config = {**config, **config.get("text_config", {})} + self.hparams["pad_token_id"] = config.get("pad_token_id") + self.hparams["eod_token_id"] = config.get("eod_token_id") + if (self.dir_model / "tokenizer.json").is_file(): tokens, toktypes, tokpre = self.get_vocab_base() self.gguf_writer.add_tokenizer_model("gpt2") @@ -199,7 +181,6 @@ def set_vocab(self): token_types = ('bos', 'eos', 'unk', 'sep', 'cls', 'mask') special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True, special_token_types=token_types) special_vocab.add_to_gguf(self.gguf_writer) - self._fix_special_tokens() else: from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) @@ -251,7 +232,18 @@ def set_vocab(self): # FIX for BOS token: Overwrite incorrect id read from config.json if self.hparams['hidden_size'] == 4096: self.gguf_writer.add_bos_token_id(127958) # only for 7b dense, fix <|bos|> token - self._fix_special_tokens() + + # Fix EOS/EOT tokens that are incorrect in upstream configs. + eod_id = self.hparams.get("eod_token_id") + if eod_id is not None: + self.gguf_writer.add_eos_token_id(eod_id) + + gen_cfg = self.dir_model / "generation_config.json" + if gen_cfg.is_file(): + with open(gen_cfg, encoding="utf-8") as f: + eos = json.load(f).get("eos_token_id") + if isinstance(eos, list) and len(eos) >= 2: + self.gguf_writer.add_eot_token_id(eos[0]) def set_gguf_parameters(self): # Some HunYuanVL variants set num_experts=1 (not real MoE); diff --git a/conversion/hy_v4.py b/conversion/hy_v4.py index f564b9ec25f2..358e21fe59ae 100644 --- a/conversion/hy_v4.py +++ b/conversion/hy_v4.py @@ -9,20 +9,6 @@ from .deepseek import DeepseekV2Model -def split_kv_b_proj(weight: torch.Tensor, n_head: int, qk_nope: int, v_head_dim: int): - """Split kv_b_proj into k_b (transposed) and v_b, matching DeepSeek MLA absorption. - - weight: [n_head*(qk_nope+v_head_dim), kv_lora_rank]. - Returns (k_b, v_b): k_b [n_head, kv_lora_rank, qk_nope], v_b [n_head, v_head_dim, kv_lora_rank]. - """ - kv_lora = weight.shape[-1] - assert weight.shape[0] == n_head * (qk_nope + v_head_dim) - kv_b = weight.view(n_head, qk_nope + v_head_dim, kv_lora) - k_b, v_b = torch.split(kv_b, [qk_nope, v_head_dim], dim=1) - k_b = k_b.transpose(1, 2).contiguous() # [n_head, kv_lora, qk_nope] - return k_b, v_b.contiguous() - - def split_gate_up(weight: torch.Tensor, moe_intermediate_size: int): """Split a fused stacked gate_up expert tensor into (gate, up). @@ -36,6 +22,7 @@ def split_gate_up(weight: torch.Tensor, moe_intermediate_size: int): @ModelBase.register("HYV4ForCausalLM") +@ModelBase.example("tencent/Hy4-preview") class HYV4Model(DeepseekV2Model): """HY_V4: DeepSeek-V3 style MLA + MoE with iHC, a gated MLA output and a learnable sink. @@ -54,6 +41,8 @@ class HYV4Model(DeepseekV2Model): model_arch = gguf.MODEL_ARCH.HY_V4 + merge_expert = False + # tensors a "full" indexer layer must carry INDEXER_SUFFIXES = frozenset({ "self_attn.indexer.wq_b.weight", @@ -186,6 +175,10 @@ def set_gguf_parameters(self): ) def prepare_tensors(self): + # Hy4-preview for some reason has num_key_value_heads equal to 8, so override it here + # without this conversion/deepseek.py fails on assert + self.hparams["num_key_value_heads"] = self.hparams["num_attention_heads"] + # validate before the base materializes tensors, so a mismatch fails early is_full = self.indexer_is_full() if is_full is not None: @@ -227,85 +220,25 @@ def tensor_force_quant(self, name, new_name, bid, n_dims): def modify_tensors(self, data_torch: torch.Tensor, name: str, bid: int | None) -> Iterable[tuple[str, torch.Tensor]]: hparams = self.hparams - n_head = hparams["num_attention_heads"] - qk_nope = hparams["qk_nope_head_dim"] - v_head_dim = hparams["v_head_dim"] moe_inter = hparams["moe_intermediate_size"] tn = self.format_tensor_name - # ---- global (non per-layer) ---- - if name == "model.embed_tokens.weight": - return [(tn(gguf.MODEL_TENSOR.TOKEN_EMBD), data_torch)] - if name == "model.norm.weight": - return [(tn(gguf.MODEL_TENSOR.OUTPUT_NORM), data_torch)] - if name == "lm_head.weight": - return [(tn(gguf.MODEL_TENSOR.OUTPUT), data_torch)] - if name == "model.hc_head.hc_head_fn": - return [(tn(gguf.MODEL_TENSOR.HC_HEAD_FN), data_torch)] - if name == "model.hc_head.hc_head_base": - return [(tn(gguf.MODEL_TENSOR.HC_HEAD_BASE), data_torch)] - if name == "model.hc_head.hc_head_scale": - return [(tn(gguf.MODEL_TENSOR.HC_HEAD_SCALE), data_torch)] - - assert bid is not None, f"expected a per-layer tensor, got {name!r}" - - # ---- per-layer, keyed by suffix after 'model.layers.{bid}.' ---- - suffix = name.split(f"model.layers.{bid}.", 1)[-1] - - # note: q_b_proj and kv_a_proj_with_mqa are mapped straight through (no RoPE permute), - # the graph rotates consecutive pairs so the rows need no reordering - simple = { - "input_layernorm.weight": (gguf.MODEL_TENSOR.ATTN_NORM, ".weight"), - "post_attention_layernorm.weight": (gguf.MODEL_TENSOR.FFN_NORM, ".weight"), - "self_attn.q_a_proj.weight": (gguf.MODEL_TENSOR.ATTN_Q_A, ".weight"), - "self_attn.q_a_layernorm.weight": (gguf.MODEL_TENSOR.ATTN_Q_A_NORM, ".weight"), - "self_attn.q_b_proj.weight": (gguf.MODEL_TENSOR.ATTN_Q_B, ".weight"), - "self_attn.kv_a_proj_with_mqa.weight": (gguf.MODEL_TENSOR.ATTN_KV_A_MQA, ".weight"), - "self_attn.kv_a_layernorm.weight": (gguf.MODEL_TENSOR.ATTN_KV_A_NORM, ".weight"), - "self_attn.o_proj.weight": (gguf.MODEL_TENSOR.ATTN_OUT, ".weight"), - "self_attn.linear_gate.weight": (gguf.MODEL_TENSOR.ATTN_GATE, ".weight"), - "self_attn.learnable_sink_param": (gguf.MODEL_TENSOR.ATTN_SINKS, ".weight"), - "self_attn.indexer.wq_b.weight": (gguf.MODEL_TENSOR.INDEXER_ATTN_Q_B, ".weight"), - "self_attn.indexer.wk.weight": (gguf.MODEL_TENSOR.INDEXER_ATTN_K, ".weight"), - "self_attn.indexer.k_norm.weight": (gguf.MODEL_TENSOR.INDEXER_K_NORM, ".weight"), - "self_attn.indexer.k_norm.bias": (gguf.MODEL_TENSOR.INDEXER_K_NORM, ".bias"), - "self_attn.indexer.weights_proj.weight": (gguf.MODEL_TENSOR.INDEXER_PROJ, ".weight"), - "hc_attn_layer.hc_pre.hc_fn": (gguf.MODEL_TENSOR.HC_ATTN_FN, ".weight"), - "hc_attn_layer.hc_pre.hc_base": (gguf.MODEL_TENSOR.HC_ATTN_BASE, ".weight"), - "hc_attn_layer.hc_pre.hc_scale": (gguf.MODEL_TENSOR.HC_ATTN_SCALE, ".weight"), - "hc_mlp_layer.hc_pre.hc_fn": (gguf.MODEL_TENSOR.HC_FFN_FN, ".weight"), - "hc_mlp_layer.hc_pre.hc_base": (gguf.MODEL_TENSOR.HC_FFN_BASE, ".weight"), - "hc_mlp_layer.hc_pre.hc_scale": (gguf.MODEL_TENSOR.HC_FFN_SCALE, ".weight"), - "mlp.gate.weight": (gguf.MODEL_TENSOR.FFN_GATE_INP, ".weight"), - "mlp.gate.e_score_correction.bias":(gguf.MODEL_TENSOR.FFN_EXP_PROBS_B, ".bias"), - "mlp.gate_proj.weight": (gguf.MODEL_TENSOR.FFN_GATE, ".weight"), - "mlp.up_proj.weight": (gguf.MODEL_TENSOR.FFN_UP, ".weight"), - "mlp.down_proj.weight": (gguf.MODEL_TENSOR.FFN_DOWN, ".weight"), - "mlp.shared_experts.gate_proj.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"), - "mlp.shared_experts.up_proj.weight": (gguf.MODEL_TENSOR.FFN_UP_SHEXP, ".weight"), - "mlp.shared_experts.down_proj.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"), - } - if suffix in simple: - key, sfx = simple[suffix] - return [(tn(key, bid, sfx), data_torch)] - - # kv_b_proj: split into k_b (transposed) and v_b - if suffix == "self_attn.kv_b_proj.weight": - k_b, v_b = split_kv_b_proj(data_torch, n_head, qk_nope, v_head_dim) - return [ - (tn(gguf.MODEL_TENSOR.ATTN_K_B, bid), k_b), - (tn(gguf.MODEL_TENSOR.ATTN_V_B, bid), v_b), - ] - # fused stacked experts: split gate_up into gate/up - if suffix == "mlp.experts.gate_up_proj": + if name.endswith("mlp.experts.gate_up_proj"): gate, up = split_gate_up(data_torch, moe_inter) - return [ - (tn(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), gate), - (tn(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), up), - ] - if suffix == "mlp.experts.down_proj": - return [(tn(gguf.MODEL_TENSOR.FFN_DOWN_EXP, bid), data_torch)] - - raise ValueError(f"Unsupported HY_V4 tensor {name!r} (suffix {suffix!r})") + yield from super().modify_tensors(gate, tn(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), bid) + yield from super().modify_tensors(up, tn(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), bid) + return + + # add .weight suffixes + if name.endswith("mlp.experts.down_proj") or name.endswith(".self_attn.learnable_sink_param"): + name += ".weight" + + if re.search(r"\.hc_head\.hc_head_(?:fn|base|scale)$", name): + name += ".weight" + + if re.search(r"\.hc_(?:attn|mlp)_layer\.hc_pre\.hc_(?:fn|base|scale)$", name): + name += ".weight" + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/conversion/kimi_k3.py b/conversion/kimi_k3.py index d15d1d64bfb9..70aabb70513b 100644 --- a/conversion/kimi_k3.py +++ b/conversion/kimi_k3.py @@ -2,15 +2,14 @@ import re from pathlib import Path -from typing import Callable, Iterable, Iterator, TYPE_CHECKING +from typing import Iterable, Iterator, TYPE_CHECKING -import numpy as np import torch if TYPE_CHECKING: from torch import Tensor -from .base import LazyTorchTensor, ModelBase, TextModel, gguf, logger +from .base import ModelBase, TextModel, gguf, logger from .kimi_linear import KimiLinearModel @@ -104,36 +103,6 @@ def dequant_model(self): "only the routed experts have a repack path" ) - def _mxfp4_expert_tensor(self, loaders: list[tuple[Callable[[], Tensor], Callable[[], Tensor]]]): - """ - One stacked [n_expert, rows, cols] MXFP4 tensor, built lazily. - - gguf_writer holds every added tensor until the final write, so building - this eagerly (like the DeepSeek-V4 path does) keeps all ~1.38 TB of - experts in memory. lazy means only the tensor being written is resident. - """ - # meta shapes, so this does not read any weights - rows, packed_cols = loaders[0][0]().shape - n_blocks = (packed_cols * 2) // 32 - byte_shape = (len(loaders), rows, n_blocks * 17) - - def load(fns: list[tuple[Callable[[], Tensor], Callable[[], Tensor]]]) -> np.ndarray: - out = np.empty(byte_shape, dtype=np.uint8) - for eid, (packed_fn, scale_fn) in enumerate(fns): - out[eid] = self.repack_mxfp4_blocks( - LazyTorchTensor.to_eager(packed_fn()), - LazyTorchTensor.to_eager(scale_fn()), - ) - return out - - # loaders goes through args, not the closure, so that `func` matches - # LazyBase's single-argument shape - return gguf.LazyNumpyTensor( - meta=gguf.LazyNumpyTensor.meta_with_dtype_and_shape(np.uint8, byte_shape), - args=(loaders,), - func=load, - ) - def _write_mxfp4_experts(self) -> None: n_experts = self.hparams["num_experts"] diff --git a/conversion/maple.py b/conversion/maple.py new file mode 100644 index 000000000000..fb0e87804d53 --- /dev/null +++ b/conversion/maple.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from typing import Iterable, TYPE_CHECKING, cast + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import LazyTorchTensor, ModelBase, TextModel, gguf + + +@ModelBase.register("MapleForCausalLM") +@ModelBase.example("deepgrove/maple-preview") +class MapleModel(TextModel): + model_arch = gguf.MODEL_ARCH.MAPLE + + def set_gguf_parameters(self): + super().set_gguf_parameters() + hparams = self.hparams + + assert hparams["hidden_act"] == "silu" + assert hparams.get("num_shared_experts", 0) == 0 + assert hparams.get("norm_topk_prob", True) + assert hparams.get("nope_on_global_attention", False) + + head_dim = hparams.get("head_dim", hparams["hidden_size"] // hparams["num_attention_heads"]) + partial_rotary_factor = self.rope_parameters.get("partial_rotary_factor", 1.0) + + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + self.gguf_writer.add_rope_dimension_count(int(head_dim * partial_rotary_factor)) + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern([layer_type == "sliding_attention" for layer_type in hparams["layer_types"]]) + self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"]) + # the reference clamps the MoE SwiGLU gate/up at 7.0 (modeling_maple.py) + self.gguf_writer.add_swiglu_clamp_exp([7.0] * self.block_count) + + _experts: list[dict[str, Tensor]] | None = None + + @staticmethod + def _stack_experts(tensors: list[Tensor]) -> Tensor: + shape = (len(tensors), *tensors[0].shape) + dtype = tensors[0].dtype + meta = LazyTorchTensor.meta_with_dtype_and_shape(dtype, shape) + + # tensors goes through args, not the closure, so that `func` matches + # LazyBase's single-argument shape + def stack(ts: list[Tensor]) -> Tensor: + result = torch.empty(shape, dtype=dtype) + for expert_id, tensor in enumerate(ts): + result[expert_id].copy_(LazyTorchTensor.to_eager(tensor)) + ts.clear() + return result + + return cast(torch.Tensor, LazyTorchTensor(meta=meta, args=(tensors,), func=stack)) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if "mlp.experts" in name: + n_experts = self.hparams["num_experts"] + assert bid is not None + + if self._experts is None: + self._experts = [{} for _ in range(self.block_count)] + + self._experts[bid][name] = data_torch + + if len(self._experts[bid]) >= n_experts * 3: + for weight_name in ("down_proj", "gate_proj", "up_proj"): + tensors = [] + + for expert_id in range(n_experts): + expert_name = f"model.layers.{bid}.mlp.experts.{expert_id}.{weight_name}.weight" + tensors.append(self._experts[bid].pop(expert_name)) + + merged_name = f"model.layers.{bid}.mlp.experts.{weight_name}.weight" + yield from super().modify_tensors(self._stack_experts(tensors), merged_name, bid) + return + + yield from super().modify_tensors(data_torch, name, bid) + + def prepare_tensors(self): + super().prepare_tensors() + + if self._experts is not None: + experts = [name for layer in self._experts for name in layer] + if experts: + raise ValueError(f"Unprocessed experts: {experts}") diff --git a/conversion/mimo.py b/conversion/mimo.py index 15dbeb7e754f..8a2689b96538 100644 --- a/conversion/mimo.py +++ b/conversion/mimo.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from torch import Tensor -from .base import MmprojModel, ModelBase, TextModel, gguf +from .base import MmprojModel, ModelBase, TextModel, gguf, logger @ModelBase.register("MiMoV2FlashForCausalLM", "MiMoV2ForCausalLM") @@ -167,6 +167,84 @@ def set_gguf_parameters(self): self.gguf_writer.add_nextn_predict_layers(self._n_nextn) + _MXFP4_EXPERT_RE = re.compile( + r"^model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate|up|down)_proj\.weight$" + ) + _MXFP4_PROJ = { + "gate": gguf.MODEL_TENSOR.FFN_GATE_EXP, + "up": gguf.MODEL_TENSOR.FFN_UP_EXP, + "down": gguf.MODEL_TENSOR.FFN_DOWN_EXP, + } + + def _is_mxfp4_packed(self) -> bool: + quant_config = self.hparams.get("quantization_config") or {} + if quant_config.get("store_dtype") != "mxfp4": + return False + # repack_mxfp4_blocks assumes ggml's 32-element group + block_size = quant_config.get("mxfp4_block_size", 32) + if block_size != 32: + raise NotImplementedError( + f"MXFP4 block size {block_size} is not ggml's QK_MXFP4 (32)") + return True + + def _write_mxfp4_experts(self) -> None: + n_experts = self.hparams["n_routed_experts"] + + # the FP8 half uses `weight_scale_inv` and is left to dequant_model + stray = [n for n in self.model_tensors + if n.endswith(".weight_scale") and not self._MXFP4_EXPERT_RE.match(n.removesuffix("_scale"))] + if stray: + raise NotImplementedError( + f"{len(stray)} MXFP4 tensor(s) outside the routed experts, e.g. {stray[0]!r}; " + "only the routed experts have a repack path" + ) + + # (bid, proj) -> {expert id: (weight name, scale name)} + groups: dict[tuple[int, str], dict[int, tuple[str, str]]] = {} + for name in self.model_tensors: + m = self._MXFP4_EXPERT_RE.match(name) + if m is None: + continue + bid, eid, proj = int(m.group(1)), int(m.group(2)), m.group(3) + scale_name = name + "_scale" + if scale_name not in self.model_tensors: + raise KeyError(f"missing {scale_name} for {name}") + groups.setdefault((bid, proj), {})[eid] = (name, scale_name) + + consumed: list[str] = [] + for (bid, proj), experts in sorted(groups.items()): + missing = [e for e in range(n_experts) if e not in experts] + if missing or len(experts) != n_experts: + raise KeyError( + f"layer {bid} {proj}_proj: {len(experts)} of {n_experts} experts present" + + (f", first missing is {missing[0]}" if missing else "") + ) + + loaders = [] + for eid in range(n_experts): + weight_name, scale_name = experts[eid] + loaders.append((self.model_tensors[weight_name], self.model_tensors[scale_name])) + consumed += [weight_name, scale_name] + + data = self._mxfp4_expert_tensor(loaders) + new_name = self.format_tensor_name(self._MXFP4_PROJ[proj], bid) + shape = gguf.quant_shape_from_byte_shape(data.shape, gguf.GGMLQuantizationType.MXFP4) + logger.info( + f"{new_name}: repacked {n_experts} experts to MXFP4, " + f"shape = {{{', '.join(str(n) for n in reversed(shape))}}}" + ) + self.gguf_writer.add_tensor(new_name, data, raw_dtype=gguf.GGMLQuantizationType.MXFP4) + + for name in consumed: + del self.model_tensors[name] + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + # not a generator on purpose: base.py chains this with get_tensors(), so the + # tensors used here must be removed from model_tensors before that starts + if self._is_mxfp4_packed(): + self._write_mxfp4_experts() + return () + _experts: list[dict[str, Tensor]] | None = None @classmethod @@ -192,7 +270,7 @@ def modify_tensors(self, data_torch, name, bid): bid = new_bid # process the experts separately - if name.find("mlp.experts") != -1: + if ".mlp.experts." in name and name.endswith(".weight"): n_experts = self.hparams["n_routed_experts"] assert bid is not None @@ -229,6 +307,10 @@ def prepare_tensors(self): if len(experts) > 0: raise ValueError(f"Unprocessed experts: {experts}") + if self._is_mxfp4_packed(): + self._is_mxfp4 = True + self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE + @ModelBase.register("MiMoV2ForCausalLM") @ModelBase.example("XiaomiMiMo/MiMo-V2.5") @@ -382,6 +464,8 @@ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: "_codebook.inited", ) for name, tensor in state_dict.items(): + if name.startswith("decoder."): + continue if name.endswith(skip_suffixes): continue if m := codebook_re.match(name): diff --git a/conversion/minimax.py b/conversion/minimax.py index 53a9ff60f836..aac340c61414 100644 --- a/conversion/minimax.py +++ b/conversion/minimax.py @@ -25,7 +25,7 @@ def _get_suppress_tokens(self) -> Sequence[int] | None: # they get in the way of the token sampling process and must be suppressed tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) - tokenizer_vocab_size = tokenizer.vocab_size + tokenizer_vocab_size = tokenizer.vocab_size # ty: ignore[unresolved-attribute] with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: weight_map = json.load(f)["weight_map"] diff --git a/conversion/muse_glimmer.py b/conversion/muse_glimmer.py index b205f70a0ebe..c86b33227366 100644 --- a/conversion/muse_glimmer.py +++ b/conversion/muse_glimmer.py @@ -37,7 +37,7 @@ def set_vocab(self): from transformers import AutoTokenizer tok = AutoTokenizer.from_pretrained(self.dir_model) - eot_id = tok.convert_tokens_to_ids("<|eot|>") + eot_id = tok.convert_tokens_to_ids("<|eot|>") # ty: ignore[unresolved-attribute] if isinstance(eot_id, int) and eot_id >= 0: self.gguf_writer.add_eot_token_id(eot_id) diff --git a/conversion/nemotron.py b/conversion/nemotron.py index c7adb2e27aca..65728d554485 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -216,14 +216,14 @@ def __init__(self, *args, **kwargs): hparams = kwargs.pop("hparams", None) if hparams is None: hparams = ModelBase.load_hparams(args[0], self.is_mistral_format) - has_moe_params = ( - "num_experts_per_tok" in hparams - or (isinstance(hparams.get("llm_config"), dict) and "num_experts_per_tok" in hparams["llm_config"]) - ) + llm_config = {**hparams, **(hparams.get("llm_config") or {})} + + has_moe_params = "num_experts_per_tok" in llm_config + layers_block_type = llm_config.get("layers_block_type") + if has_moe_params: self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE self.is_moe = True - layers_block_type = hparams.get("layers_block_type") if layers_block_type is not None: hparams["num_hidden_layers"] = len(layers_block_type) diff --git a/conversion/qwen.py b/conversion/qwen.py index 3c52628b3c14..4536275e3f7e 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from torch import Tensor -from .base import LazyTorchTensor, ModelBase, TextModel, gguf, logger +from .base import LazyTorchTensor, ModelBase, ModelType, TextModel, get_model_architecture, gguf, logger @ModelBase.register("QWenLMHeadModel") @@ -379,6 +379,13 @@ def set_gguf_parameters(self): self.gguf_writer.add_ssm_group_count(self.hparams["linear_num_key_heads"]) self.gguf_writer.add_ssm_time_step_rank(self.hparams["linear_num_value_heads"]) self.gguf_writer.add_ssm_inner_size(self.hparams["linear_value_head_dim"] * self.hparams["linear_num_value_heads"]) + if (layer_types := self.hparams.get("layer_types")) is not None: + n_layer = self.hparams["num_hidden_layers"] + if len(layer_types) != n_layer: + raise ValueError(f"layer_types has {len(layer_types)} entries, expected num_hidden_layers ({n_layer})") + recurrent = [t == "linear_attention" for t in layer_types] + recurrent += [False] * (self.block_count - n_layer) + self.gguf_writer.add_recurrent_layers(recurrent) self.gguf_writer.add_full_attention_interval(self.hparams.get("full_attention_interval", 4)) if (rope_dim := self.hparams.get("head_dim")) is None: rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] @@ -667,7 +674,7 @@ def set_vocab(self): from . import get_model_class with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f: target_hparams = json.load(f) - target_arch = target_hparams["architectures"][0] + target_arch = get_model_architecture(target_hparams, ModelType.TEXT) target_cls = get_model_class(target_arch) if target_cls is not type(self): @@ -714,7 +721,7 @@ def set_gguf_parameters(self): if embedding_scale is not None: self.gguf_writer.add_embedding_scale(float(embedding_scale)) - target_layer_ids = dflash_config.get("target_layer_ids", []) + target_layer_ids = dflash_config.get("target_layer_ids", self.hparams.get("target_layer_ids", [])) if target_layer_ids: extract_layer_ids = [i + 1 for i in target_layer_ids] self.gguf_writer.add_target_layers(extract_layer_ids) @@ -722,8 +729,9 @@ def set_gguf_parameters(self): use_sliding_window = self.hparams.get("use_sliding_window", False) or dflash_config.get("use_swa", False) sliding_window = dflash_config.get("swa_window_size") or self.hparams.get("sliding_window") layer_types = self.hparams.get("layer_types") - if use_sliding_window and sliding_window and layer_types: - is_swa = [lt == "sliding_attention" for lt in layer_types] + if use_sliding_window and sliding_window: + is_swa = ([True] * self.block_count if dflash_config.get("use_swa", False) + else [lt == "sliding_attention" for lt in layer_types or []]) self.gguf_writer.add_sliding_window(sliding_window) self.gguf_writer.add_sliding_window_pattern(is_swa) @@ -899,13 +907,6 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca return None return super().filter_tensors(item) - _ROPE_PERMUTE_SUFFIXES = ( - "self_attn.q_proj.weight", - "self_attn.k_proj.weight", - "self_attn.q_norm.weight", - "self_attn.k_norm.weight", - ) - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if name == "model.d2t": self._d2t = data_torch @@ -914,12 +915,6 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith("lm_head.weight"): return - # interleaved-rope checkpoints (rope_is_neox_style = false) -> NeoX layout: per head, even dims first then odd - if not self.hparams.get("rope_is_neox_style", True) and name.endswith(self._ROPE_PERMUTE_SUFFIXES): - head_dim = self.hparams["head_dim"] - shape = data_torch.shape - data_torch = data_torch.reshape(-1, head_dim // 2, 2, *shape[1:]).transpose(1, 2).reshape(shape) - yield from super().modify_tensors(data_torch, name, bid) def prepare_tensors(self): diff --git a/conversion/spark2_5.py b/conversion/spark2_5.py new file mode 100644 index 000000000000..44a0bd262e6a --- /dev/null +++ b/conversion/spark2_5.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from collections.abc import Iterable +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, gguf + + +@ModelBase.register("Spark2_5ForCausalLM") +@ModelBase.example("XHToken/Spark-X2.5-1.7B") +class Spark2_5Model(TextModel): + model_arch = gguf.MODEL_ARCH.SPARK2_5 + + def set_gguf_parameters(self) -> None: + super().set_gguf_parameters() + + hparams = self.hparams + layer_types = hparams["layer_types"] + if len(layer_types) != self.block_count: + raise ValueError( + f"Spark2_5 layer_types length {len(layer_types)} != num_hidden_layers {self.block_count}" + ) + if any(layer_type not in ("sliding_attention", "full_attention") for layer_type in layer_types): + raise ValueError(f"Spark2_5 has unsupported layer_types: {layer_types}") + if hparams.get("gate_attn_act_mode") != "sigmoid" or hparams.get("headwise_attn_output_gate") is not True: + raise ValueError("Spark2_5 conversion requires head-wise sigmoid attention gates") + if hparams.get("hidden_act") != "gelu": + raise ValueError(f"Spark2_5 conversion requires GELU, got {hparams.get('hidden_act')!r}") + + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern( + [layer_type == "sliding_attention" for layer_type in layer_types] + ) + + head_dim = hparams["head_dim"] + full_rope = self.rope_parameters["full_attention"] + swa_rope = self.rope_parameters["sliding_attention"] + self.gguf_writer.add_rope_dimension_count( + int(head_dim * float(full_rope["partial_rotary_factor"])) + ) + self.gguf_writer.add_rope_dimension_count_swa( + int(head_dim * float(swa_rope["partial_rotary_factor"])) + ) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name.endswith(".self_attn.q_k_v_proj.weight"): + if bid is None: + raise ValueError(f"Spark2_5 fused QKV tensor has no block id: {name}") + yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid), data_torch + return + + if name.endswith(".self_attn.g_proj.weight"): + if bid is None: + raise ValueError(f"Spark2_5 attention gate tensor has no block id: {name}") + expected = self.hparams["num_attention_heads"] + if data_torch.shape[0] != expected: + raise ValueError( + f"Spark2_5 layer {bid} attention gate width {data_torch.shape[0]} != head count {expected}" + ) + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 78ad26c65630..e09616b190cf 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -157,6 +157,10 @@ def parse_args() -> argparse.Namespace: help="Store tensors dequantized from FP8 as Q8_0 instead of BF16/F16.", ) + parser.add_argument( + "--fuse-qkv", action="store_true", + help="Fuse separate Q, K, V weight tensors into a single QKV tensor.", + ) parser.add_argument( "--target-model-dir", type=str, default=None, help=( @@ -290,6 +294,7 @@ def main() -> None: target_model_dir=Path(args.target_model_dir) if args.target_model_dir else None, fuse_gate_up_exps=args.fuse_gate_up_exps, fp8_as_q8=args.fp8_as_q8, + fuse_qkv=args.fuse_qkv, ) if args.vocab_only: diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index c4141afa6142..24b9bc075777 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -163,6 +163,7 @@ class TOKENIZER_TYPE(IntEnum): {"name": "granite-embed-multi-311m", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ibm-granite/granite-embedding-311m-multilingual-r2", }, {"name": "mellum2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Base"}, {"name": "laguna", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/poolside/Laguna-XS.2", }, + {"name": "ufakzeka", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ufakai/ufakzeka-1", }, ] # some models are known to be broken upstream, so we will skip them as exceptions @@ -191,6 +192,11 @@ class TOKENIZER_TYPE(IntEnum): {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"}, # lfm2 variants {"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"}, + # hrm-text (DFM Mimir) is SPM-style BPE: normalizer maps ' ' -> '▁', merges + # over the whole text (fix_mistral_regex inserts a tekken regex that is a + # no-op here); the gemma4 pre (escape ws, split on newlines only) matches it. + {"name": "gemma4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/danish-foundation-models/DFM-Mimir", "chkhsh": "846deafc5b0fa786186fa4ae6c7b49903cf2f1d1895bdb80b9120d60be135252"}, + {"name": "spark2_5", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/XHToken/Spark-X2.5-1.7B", "chkhsh": "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed"}, ] diff --git a/docs/autoparser.md b/docs/autoparser.md index b5e32621df56..2a7ea00b4f03 100644 --- a/docs/autoparser.md +++ b/docs/autoparser.md @@ -514,6 +514,7 @@ The following templates have active tests in `tests/test-chat.cpp`: | Mistral Small 3.2 | JSON_NATIVE | `[TOOL_CALLS]func[ARGS]{...}` with call ID | | Devstral | JSON_NATIVE | `[TOOL_CALLS]func[ARGS]{...}` without call ID | | StepFun 3.5 Flash | TAG_WITH_TAGGED | `<function=X><parameter=Y>` format | +| Spark2.5 | TAG_WITH_TAGGED | `<tool_call>name<arg_key>...<arg_value>...` format | ## Adding Support for New Templates diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index 9b43807d36b3..3d791977559e 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -12,6 +12,8 @@ The OpenVINO backend is implemented in `ggml/src/ggml-openvino` and provides a t - Compiles and caches the model for the target device. - Binds GGML tensor memory to OpenVINO inference tensors and runs inference. +For guidance on contributing to the OpenVINO backend, see the [OpenVINO Backend Contributing Guide](https://github.com/ravi9/llamacpp-ov-dev-guide/blob/main/contributing-llamacpp-ov.md). + ## Contents - [Supported Devices](#supported-devices) @@ -96,7 +98,7 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_ - **SL** = Stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`) - **SF** = Stateful (`GGML_OPENVINO_STATEFUL_EXECUTION=1`) - Note: The NPU operates in stateless mode only. -- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel OpenCL GPU Driver 26.31.39395.13-0 | Intel NPU Driver 1.35.0. +- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel Graphics Compiler 2.41.5 | Intel OpenCL GPU Driver 26.31.39395.13-0 | Intel NPU Driver 1.38.0. - See [Known Limitations](#known-limitations) for context on observed failures. | Model | CPU (SL / SF) | GPU (SL / SF) | NPU (SL) | @@ -117,9 +119,9 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_ | [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ | | | | | | | [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | -| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ | -| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ | -| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ | +| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ | +| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | +| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ | | | | | | | [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | | [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | @@ -132,9 +134,9 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_ | [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | | [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | | | | | | -| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ | +| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | | [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | -| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ / ✓ | ✗ / ✗ | ✗ | +| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ | | [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | | | | | | | [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | @@ -242,8 +244,8 @@ chmod +x build-llamacpp-ov.sh # ============================================ set -euo pipefail -OPENVINO_VERSION_MAJOR="2026.3.1" -OPENVINO_VERSION_FULL="2026.3.1.22476.56d9685302d" +OPENVINO_VERSION_MAJOR="2026.4" +OPENVINO_VERSION_FULL="2026.4.0.22959.99c81491cc3" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" OPENVINO_INSTALL_DIR="/opt/intel/openvino_${OPENVINO_VERSION_MAJOR}" @@ -340,7 +342,7 @@ echo " ./build/ReleaseOV/bin/llama-cli -m model.gguf" ``` > [!NOTE] -> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. +> The script pins OpenVINO `2026.4` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. </details> @@ -370,8 +372,8 @@ REM ============================================ REM llama.cpp OpenVINO Build Script (Ninja) REM ============================================ -set "OPENVINO_VERSION_MAJOR=2026.3.1" -set "OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d" +set "OPENVINO_VERSION_MAJOR=2026.4" +set "OPENVINO_VERSION_FULL=2026.4.0.22959.99c81491cc3" set "SCRIPT_DIR=%~dp0" set "VCPKG_DIR=C:\vcpkg" @@ -550,7 +552,7 @@ endlocal ``` > [!NOTE] -> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**. +> The script pins OpenVINO `2026.4` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**. </details> @@ -719,10 +721,13 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Enable stateful KV cache for better performance. Recommended on CPU, GPU. | | `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. | +| `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. | | `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. | | `GGML_OPENVINO_MEMORY_OPTIMIZE` | Boolean | `0` | Umbrella switch for compile-time memory reductions. Enables `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS` unless those fine-grained variables are explicitly set. | | `GGML_OPENVINO_REDUCE_COMPILE_MEM`| Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` | Reduce compile-time host memory use by streaming weight requantization and avoiding extra weight-node materialization where possible. Set explicitly to override the umbrella switch. | | `GGML_OPENVINO_RELEASE_WEIGHTS` | Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` on GPU | GPU-only. Release host weight buffers after the compiled model cache can reuse the device/plugin copy. Requires stable graph shapes; dynamic workloads that need recompilation should leave this disabled. | +| `GGML_OPENVINO_SPILL_DIR` | String | `not set` | Directory for a disk-backed weight buffer. When set, the repacked weight buffer is mapped from an unlinked file on this path instead of anonymous memory, so its pages are reclaimable under memory pressure instead of staying pinned, cutting the load-time host memory peak. Must point at real storage; a tmpfs mount (e.g. `/tmp` on many systems) backs it with RAM and makes the peak worse. | +| `GGML_OPENVINO_REQUANT_KQUANT` | String | `not set` | Requantize Q6_K/Q5_K weights (and matching MoE expert weights) to a 4-bit target instead of the default Q8_0_C, trading accuracy for less memory traffic. One of `q4_sym128` (Q6_K/Q5_K only), `q4_sym128_all` (Q4_K too, drops its per-group zero point), `q4_asym64_all` (Q6_K/Q5_K/Q4_K, keeps a real zero point at group 64), or `native` (no requantization). | | `GGML_OPENVINO_PROFILING` | Boolean | `0` | Enable execution-time profiling. | | `GGML_OPENVINO_DUMP_CGRAPH` | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. | | `GGML_OPENVINO_DUMP_IR` | Boolean | `0` | Serialize OpenVINO IR files with timestamps. | diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index c4dcb02eecb5..91b2097419ed 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -790,14 +790,15 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm | Name | Value | Function | |-------------------|------------------|---------------------------------------------------------------------------------------------------------------------------| -| GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function by macro: GGML_SYCL_DEBUG | +| GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function: GGML_SYCL_DEBUG() for common debug. | +| GGML_SYCL_DEV_DEBUG | 0 (default) or 1 | Enable log function: GGML_SYCL_DEV_DEBUG() for developmental purposes by replacing GGML_SYCL_DEBUG() in special codes. Restore to GGML_SYCL_DEBUG() before committing code.| | GGML_SYCL_DEV2DEV_MEMCPY | 0 (default), 1, 2 | Choose the method of dev2dev memory copy.<br>Value: <br>* 0: SYCL API (default), only support dGPUs.<br>* 1: L0 API -- Better performance, only support dGPUs, found to lead to abnormal crash in some case. <br>* 2: Host Forward -- Most stable method for all cases (including iGPU + dGPU*N), but with lower performance (-2% to -5%).<br>SYCL & L0 API are easy to be impacted by Intel GPU driver issue. When you meet the garbled output or crash issues in multiple GPUs case, try with this debug flag to work around or check the issue.| | GGML_SYCL_ENABLE_FLASH_ATTN | 1 (default) or 0| Enable Flash-Attention. It can reduce memory usage. The performance impact depends on the LLM.| | GGML_SYCL_ENABLE_OPT | 0 or 1 (default)| Enable optimize features for Intel GPUs. (Recommended to 0 for Intel devices older than Gen 10) | | GGML_SYCL_ENABLE_GRAPH | 0 (default) or 1 | Enable running computations through SYCL Graphs feature. Disabled by default because SYCL Graph is still on development, no better performance. | | GGML_SYCL_ENABLE_HOST_PINNED_MEM | 0 or 1 (default) | Enable host pinned memory to speed up copy data from host to device. When disable it, host memory will common malloc() on CPU. Disable it when use `--load-model mlock`.| | GGML_SYCL_HOST_PINNED_MEM_2G | 0 (default) or 1 | Limit the max memory allocation to be no more than 2GB when enable host pinned memory. USM allocations above 2 GiB take the relaxed/large-allocation path, which serializes H2D copies with compute and prevents copy/compute overlap. It will impact the startup time. Need more test. Depend on `GGML_SYCL_ENABLE_HOST_PINNED_MEM=1`.| -| GGML_SYCL_GET_MEM_API | 0 (default) or 1 | Set to get memory info (free, total) by Level Zero or SYCL API:<br>0 - Level Zero API: support more GPUs, only run on Level Zero running time. When there is an error, fallback to call SYCL API. Depend on GGML_SYCL_SUPPORT_LEVEL_ZERO_API.<br>1 - SYCL API: legacy, support more running time, it can't get the free size of some GPUs (like Arc770). In such case, return total size for free size.| +| GGML_SYCL_GET_MEM_API | 0 (default) or 1 | Set to get memory info (free, total) by Level Zero or SYCL API:<br>0 - Level Zero API: support more GPUs, only run on Level Zero running time. When there is an error, fallback to call SYCL API. Depend on GGML_SYCL_SUPPORT_LEVEL_ZERO_API.<br>1 - SYCL API: legacy, support more running time, it can't get the free size of some GPUs (like Arc770). In such case, return the free size as value of total size.| | GGML_SYCL_USE_LEVEL_ZERO_API | 1 (default) or 0 | Use Level Zero API for device memory allocation instead of SYCL. Reduces system RAM usage on Intel dGPUs by avoiding DMA-buf/TTM host memory staging. Requires GGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON at build time. SYCL backend always runs on Level Zero running time even if it's set as OFF (The SYCL api will be usage for memory allocation).| | GGML_SYCL_ENABLE_DNN | 0 or 1 (default)| Enable running computations through oneDNN and always use oneMKL. | | GGML_SYCL_FA_ONEDNN | 1 (default) or 0 | Enable the oneDNN fused SDPA (flash-attention) path on supported GPUs. Set to 0 to always use the native SYCL flash-attention kernel. | @@ -805,6 +806,8 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm | GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. | | GGML_SYCL_ENABLE_MKL_FA | 1 (default) or 0 | Enable oneMKL GEMM flash attention for XMX-accelerated prompt processing with quantized KV cache. Automatically activates during prefill (prompt processing) when all conditions are met: (1) flash-attn enabled (`-fa` or `--flash-attn on`), (2) KV cache quantized (`--cache-type-k q8_0 --cache-type-v q8_0` or other `*_0/*_1` types), (3) batch size ≥ 1024 (`--batch-size 1024`), (4) prompt length ≥ 1024 tokens. Set to 0 to force the TILE kernel for A/B testing. Example minimum command: `llama-cli -m model.gguf -fa -ngl 99 --cache-type-k q8_0 --cache-type-v q8_0 --batch-size 1024 -p "your prompt"` | | GGML_SYCL_MKL_FA_DEBUG | 0 (default) or 1 | Enable per-call diagnostic logging for MKL flash attention: GEMM/softmax timings, interleaved-head detection, and buffer memory usage. | +| GGML_SYCL_MEMTRACE | 0 (default), 1, 2 | Enable record and output memory allocation diagnostics. Requires `-lv 4`. <br>0 - Disable<br>1 - Basic memory info, including current and peak allocations, as well allocations from other sources, around 50 lines per model load.<br>2 - More verbose, logging around 900 specific allocations and deallocations. | +| GGML_SYCL_MEMTRACE_STEP | 64 (default) or positive integer | With GGML_SYCL_MEMTRACE=1, the minimum growth in memory usage to trigger another log record. | | GGML_SYCL_MKL_FA_DIAG | 0 (default) or 1 | Enable output fingerprinting for MKL flash attention. Dumps the first 64 float output values for the first 6 FA calls with n_kv ≥ 1024, labeled with kernel type (MKL/TILE/VEC) for cross-kernel comparison. | | GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute. Unsupported types and layouts fall back to the standalone op kernels. See `ggml_sycl_can_fuse()`. | | GGML_SYCL_ENABLE_ESIMD | 0 or 1 (default)| Enable ESIMD kernels when available. | diff --git a/docs/backend/snapdragon/README.md b/docs/backend/snapdragon/README.md index 391c8bf230f0..ca79ca8852fc 100644 --- a/docs/backend/snapdragon/README.md +++ b/docs/backend/snapdragon/README.md @@ -188,7 +188,7 @@ llama_memory_breakdown_print: | - Host | 439 = Op test for MUL_MAT: ``` -~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --hex-hostbuf 0 --devices HTP0:0 -- test-backend-ops -b HTP0:0 -o MUL_MAT +~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --devices HTP0:0 -- test-backend-ops -b HTP0:0 -o MUL_MAT ... Backend 2/3: HTP0:0 Device description: Hexagon @@ -213,14 +213,109 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v | llama 1B Q4_0 | 729.75 MiB | 1.24 B | HTP | 99 | 4 | 128 | 0 | tg64 | 51.54 ± 1.13 | ``` +## Multi-Device Execution Modes + +The Hexagon backend supports multiple execution and partitioning modes to accommodate different model sizes, memory +constraints, and single- or multi-NPU hardware topologies: + +### 1. Single-Device Mode with Dynamic Buffer Mapping + +Runs the model on a single NPU session (e.g. `HTP0` or `HTP0:0`). + +A single NPU session provides ~3.5GB of available virtual address space. For models larger than 3.5GB, the backend +automatically maps and unmaps weight buffers during graph execution. This allows large models to run on a single NPU +without manual configuration: + +```bash +./scripts/snapdragon/run.py --target adb --devices HTP0:0 -- \ + llama-cli -m models/Llama-3.2-3B-Instruct-Q4_0.gguf -ngl 99 -p "Hello" +``` + +### 2. Layer-Split Mode across Virtual Sessions (`HTP0,HTP1,...` or `HTP0:0,HTP0:1,...`) + +Partitions model layers at load time across multiple virtual sessions hosted on a single physical NPU. + +Each virtual session acts as an independent backend device from llama.cpp's perspective (similar to multiple GPUs). +Because layers are permanently distributed across sessions, each session's allocated weights remain within its private 3.5GB +address space window, eliminating runtime buffer re-mapping overhead. + +Here is an example of running the GPT-OSS-20B model on a Snapdragon device using 4 virtual sessions on a single NPU: + +```bash +./scripts/snapdragon/run.py --target adb \ + --devices HTP0:0,HTP0:1,HTP0:2,HTP0:3 -- \ + llama-cli --load-mode none -m /data/local/tmp/gguf/gpt-oss-20b-Q4_0.gguf -t 4 \ + --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 -no-cnv -f surfing.txt +``` + +Log output snippet: + +``` +... +llama_model_loader: - type f32: 289 tensors +llama_model_loader: - type q4_0: 96 tensors +llama_model_loader: - type q8_0: 2 tensors +llama_model_loader: - type mxfp4: 72 tensors +... +load_tensors: offloaded 25/25 layers to GPU +load_tensors: CPU model buffer size = 1182.09 MiB +load_tensors: HTP0:1 model buffer size = 2512.58 MiB +load_tensors: HTP0:3 model buffer size = 2093.83 MiB +load_tensors: HTP0:0 model buffer size = 2931.34 MiB +load_tensors: HTP0:2 model buffer size = 2512.58 MiB +... +llama_perf_context_print: prompt eval time = 3843.67 ms / 197 tokens ( 19.51 ms per token, 51.25 tokens per second) +llama_perf_context_print: eval time = 1686.13 ms / 31 runs ( 54.39 ms per token, 18.39 tokens per second) +llama_perf_context_print: total time = 6266.30 ms / 228 tokens +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - HTP0:0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:2 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:3 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - Host | 1476 = 1208 + 105 + 162 | +``` + +### 3. Tensor-Split Mode across Physical Devices (`HTP0:0,HTP1:0,...`) + +Distributes model tensors across distinct physical NPU hardware cores using llama.cpp's tensor parallelism +(`--split-mode tensor`). + +Tensors are partitioned across physical NPUs for parallel execution (proportions are distributed equally by default without +needing an explicit `--tensor-split` option): + +```bash +./scripts/snapdragon/run.py --target adb \ + --devices HTP0:0,HTP1:0 -- \ + llama-cli -m models/Llama-3.2-3B-Instruct-Q4_0.gguf --split-mode tensor -ngl 99 -p "Hello" +``` + +### 4. Row-Split Multi-Device Mode via Device Grouping (`HTP0[0-1]`) + +Groups multiple physical NPU cores into a single logical device using bracket notation (`HTP0[0-1]` or `HTP0[0,1]`). + +Unlike host-level tensor-splitting, row-splitting is executed entirely inside the Hexagon backend: + +```bash +./scripts/snapdragon/run.py --target adb \ + --devices 'HTP0[0-1]' -- \ + llama-cli -m models/Llama-3.2-3B-Instruct-Q4_0.gguf -ngl 99 -p "Hello" +``` + +You can also combine row-splitting with layer-splitting across multiple grouped devices (e.g. `--devices 'HTP0[0-1],HTP1[2-3]'` +on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs using virtual sessions 0 and 1). + ## Environment variables - `GGML_HEXAGON_DEVICES` (default: not set, defaults to HTP0 session) - Controls which NPU devices and sessions to allocate. Can be configured as: - - A single integer `N`: Allocates `N` sessions named `HTP0`, `HTP1`, ..., `HTP<N-1>` (behaves identically to `GGML_HEXAGON_NDEV=N`). - - A comma-separated list of device names in `HTP<physical_idx>:<virtual_idx>` format (or legacy `HTP<idx>` format). For example, `HTP0:0,HTP0:1` creates two virtual - sessions on the first physical NPU (useful for memory limits). `HTP0:0,HTP1:0` allocates one session on each of the two physical NPUs - on a dual-NPU device. + Controls which NPU devices and sessions to allocate. Configurable via `--devices` in `run.py`: + - `N` (single integer): Allocates `N` virtual sessions named `HTP0`, `HTP1`, ..., `HTP<N-1>` on physical NPU 0. + - `HTP<phys>:<virt>,...`: Comma-separated list of individual devices specifying physical and virtual index: + - `HTP0:0,HTP0:1`: Two virtual sessions on physical NPU 0 (layer-split on single NPU). + - `HTP0:0,HTP1:0`: One session on physical NPU 0 and one on physical NPU 1 (tensor-split across physical cores). + - `HTP<name>[<phys_spec>]`: Device grouping syntax for row-split multi-device execution: + - `HTP0[0-1]`: A single logical device `HTP0` that groups physical cores 0 and 1. + - `HTP0[0-1],HTP1[2-3]`: Two layer-split devices across 4 physical NPUs (cores 0-1 and 2-3). + - `HTP0[0-1:0],HTP1[0-1:1]`: Two layer-split devices across 2 physical NPUs using virtual sessions 0 and 1. - `GGML_HEXAGON_NDEV` (deprecated) Replaced by `GGML_HEXAGON_DEVICES`. Controls the number of virtual sessions to allocate on physical NPU `0`. @@ -229,9 +324,11 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v - `GGML_HEXAGON_NHVX=0` Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version). -- `GGML_HEXAGON_HOSTBUF=1` - Controls whether the Hexagon backend allocates host buffers. By default, all buffers except for REPACK are host buffers. - This option is required for testing Ops that require REPACK buffers (MUL_MAT and MUL_MAT_ID). +- `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled) + Enables allocating host buffers for debugging. By default, host buffers are disabled. + +- `GGML_HEXAGON_DMA64=0` (default: enabled on v81+) + Disables 64-bit DMA for model weights. Set to `1` to enable it explicitly on a supported architecture. - `GGML_HEXAGON_VERBOSE=1` Enables verbose logging of Ops from the backend. Example output: @@ -246,23 +343,26 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v ``` - `GGML_HEXAGON_PROFILE=1` - Enables Op profiling: + Enables Op profiling (configurable via `--hex-profile` in `run.py`): - - `1` Basic profile with per-op `usecs` and `cycles` counters - - `2` Extended profile with per-op `usecs`, `cycles` and default PMU counter data - - `0x1,...,0x8` Extended profile with per-op `usecs`, `cycles` and custom PMU counter data + - `1`: Basic profile with per-op `usecs` and `cycles` counters + - `2`: Extended profile with per-op `usecs`, `cycles` and default PMU counter data + - `0x1,...,0x8`: Extended profile with per-op `usecs`, `cycles` and custom PMU counter data - The logging output can be either saved into a file for post-processing or it can be piped directly into the post-processing tool - to generate the report. - Examples: + The logging output can be saved to a file or piped directly into the post-processing script: - `GGML_HEXAGON_PROFILE=1 ./scripts/snapdragon/run.py --target adb -- llama-cli ... |& ./scripts/snapdragon/ggml-hexagon-profile.py -` + ```bash + ./scripts/snapdragon/run.py --target adb --hex-profile 1 -- llama-cli ... |& \ + ./scripts/snapdragon/ggml-hexagon-profile.py - + ``` - `GGML_HEXAGON_OPFILTER=regex` - Allows filtering (disabling) Ops that match the regex pattern: + Filters (disables) Ops matching the regex pattern (configurable via `--hex-opfilter` in `run.py`): - Examples: - - `GGML_HEXAGON_OPFILTER="FLASH_ATTN_EXT" ./scripts/snapdragon/run.py --target adb -- llama-cli ...` - Disable Flash Attention on Hexagon (falls back to CPU or GPU) - `GGML_HEXAGON_OPFILTER="ADD\|SUB" ./scripts/snapdragon/run.py --target adb -- llama-cli ...` - Disable ADD and SUB on Hexagon (fall back to CPU or GPU) + ```bash + # Disable Flash Attention on Hexagon (falls back to CPU or GPU) + ./scripts/snapdragon/run.py --target adb --hex-opfilter "FLASH_ATTN_EXT" -- llama-cli ... + # Disable ADD and SUB on Hexagon (fall back to CPU or GPU) + ./scripts/snapdragon/run.py --target adb --hex-opfilter "ADD|SUB" -- llama-cli ... + ``` diff --git a/docs/backend/snapdragon/developer.md b/docs/backend/snapdragon/developer.md index d7d9f2a2790d..378d47653a0c 100644 --- a/docs/backend/snapdragon/developer.md +++ b/docs/backend/snapdragon/developer.md @@ -2,16 +2,16 @@ ## Backend libraries -The Hexagon backend consist of two parts: +The Hexagon backend consists of two parts: - `libggml-hexagon` - This is the regular CPU-side GGML backend library, either shared or statically linked + This is the regular CPU-side GGML backend library, either shared or statically linked. - `libggml-htp-vNN` This is the NPU-side (HTP stands for Hexagon Tensor Processor) shared library that contains the Op dispatcher and kernels. The correct library is selected automatically at runtime based on the HW version. -Here is an example of the build artifacts +Here is an example of the build artifacts: ``` ~/src/llama.cpp$ ls -l pkg-adb/llama.cpp/lib/libggml* @@ -26,75 +26,329 @@ pkg-adb/llama.cpp/lib/libggml-htp-v81.so ## Memory buffers -Hexagon NPU backend takes advantage of the Snapdragon's unified memory model where all buffers are fully accessible by the CPU and GPU. -The NPU does have a dedicated tightly-coupled memory called VTCM but that memory is used only for intermediate data (e.g. dynamically -quantized tensors) or temporary data (chunks of the weight tensors fetched via DMA). - -Please note that currently the Hexagon backend does not implement SET/GET_ROWS Ops because there is no advantage in offloading those -to the NPU at this point. - -The backend does allocates non-host buffers for the tensors with datatypes that require repacking: Q4_0, Q8_0, MXFP4. -From the MMU perspective these buffers are still regular buffers (normal access by the CPU) they are marked as non-host simply to force -the repacking. +The Hexagon NPU backend takes advantage of Snapdragon unified memory where all DDR buffers are accessible by CPU, GPU, and NPU. +The NPU has dedicated tightly-coupled memory called VTCM (Vector Tightly-Coupled Memory). VTCM is used for intermediate data (such as +dynamically quantized activations) and streaming buffers (chunks of weight and activation tensors fetched via DMA). ## Large model handling -Hexagon NPU sessions (aka Process Domains (PD) in the Hexagon SDK) are limited to a maximum memory mapping window of around 3.5GB. +Hexagon NPU sessions have a 32-bit virtual address space window of around 3.5GB. In llama.cpp/GGML, each Hexagon session is mapped to a single GGML backend device (e.g., `HTP0:0`, `HTP0:1`, etc. when using `GGML_HEXAGON_DEVICES`, or `HTP0`, `HTP1` in legacy mode). -To support running models larger than 3.5GB on a single device, the Hexagon backend dynamically maps and unmaps execution buffers -during the graph execution cycle to stay within the Process Domain window. This enables large models to run successfully on a single -NPU device. +To support running models larger than 3.5GB on a single device, the Hexagon backend dynamically maps and unmaps buffers: +- Buffers are allocated in shared DDR (RPCMEM) via file descriptors (`fastrpc_mmap` using `FASTRPC_MAP_FD_DELAYED`). +- Pinned buffers (such as KV cache and active compute buffers) remain mapped throughout execution. +- Inactive weight buffers are dynamically mapped into the NPU session via `HAP_mmap()` during batch buffer preparation + (`prep_op_bufs()` in `htp/main.c`) and unmapped via `htp_iface_munmap()` when no longer needed by the active batch. +- This dynamic sliding window allows a single NPU session to execute models that exceed the 3.5GB window. + +Alternatively, users can partition and split the model across multiple virtual sessions or physical NPUs using layer-splitting, +tensor-splitting, or row-splitting modes. For user-facing execution modes and examples, see the +[Snapdragon user guide](README.md#multi-device-execution-modes). + +## Op and Kernel Development Guidelines + +Writing high-performance operators for Hexagon requires following specific guidelines. + +### DDR -> DMA -> VTCM Execution Pipeline + +- Strongly prefer the `DDR -> DMA -> VTCM -> compute (HVX/HMX) -> VTCM -> DMA -> DDR` data flow. +- Direct HVX reads/writes from/to DDR are less efficient and should only be used as a fallback. +- The DMA queue is a strict FIFO where operations must be pushed and popped in strict order. +- Follow the pipelined multi-buffering sequence properly (typically 2x to 16x buffering) so every push has a corresponding pop: + + 1. In the prologue, push initial DDR -> VTCM transfers to prime the pipeline. + 2. In the loop body, wait for buffer N via DMA pop, launch HVX/HMX compute on buffer N, push VTCM -> DDR writeback of result N, + and push DDR -> VTCM prefetch of buffer N+2. + 3. In the epilogue, pop all remaining in-flight transfers to drain the pipeline. + +- Because every push must be matched by a pop, `dma_queue_flush()` is not required when the pipeline sequence is followed + properly. Flushing is only used in rare exceptions where a batch of operations is pushed without individual pops. +- Use the DMA queue interface from [`dma-queue.h`](../../../ggml/src/ggml-hexagon/htp/dma-queue.h) + (`dma_queue_push_ddr_to_vtcm`, `dma_queue_pop`, `dma_queue_push_vtcm_to_ddr`). + See [`cumsum-ops.c`](../../../ggml/src/ggml-hexagon/htp/cumsum-ops.c) and + [`act-ops.c`](../../../ggml/src/ggml-hexagon/htp/act-ops.c) for reference implementations. + +### Avoid Scalar Reads and Writes to VTCM + +- Access VTCM data using DMA transfers or HVX/HMX vector instructions rather than scalar reads and writes. + +### Avoid Scalar Division in Inner Loops + +- Hexagon cores do not have hardware division instructions. +- For recurring divisions across iterations or threads, use `fastdiv` from + [`hex-fastdiv.h`](../../../ggml/src/ggml-hexagon/htp/hex-fastdiv.h) with precomputed divisors (such as + `octx->ctx->mdev.count_div` or `octx->n_threads_div`). +- Do not call `init_fastdiv_values()` for single-use divisions; use standard compiler division (`/`) instead. + +### Host-Side Precomputation via `kernel_params` + +- Precompute tensor shapes, strides, scale conversions, tiling layouts, and validation checks on the host CPU during graph + preparation in [`ggml-hexagon.cpp`](../../../ggml/src/ggml-hexagon/ggml-hexagon.cpp). +- Pack precomputed parameters into the operator's fixed `kernel_params` structure in `htp_op_node` (such as + `htp_mm_kernel_params`, `htp_unary_kernel_params`, `htp_fa_kernel_params`, `htp_get_rows_kernel_params`). +- The NPU executes directly using `octx->kernel_params` without redundant runtime metadata extraction or validation. +- **Strict Host-Kernel Alignment**: + - Verify that parameters calculated by the host CPU are strictly honored by the NPU kernel. + - Ensure the kernel does not ignore host-computed fields (for example, falling back to `octx->n_threads` instead of + using `kparams->n_threads`, or ignoring precomputed `tasks_per_thread` and chunk counts). + - Both human developers and coding agents must audit both sides of the interface: ensure fields populated in `kernel_params` + in [`ggml-hexagon.cpp`](../../../ggml/src/ggml-hexagon/ggml-hexagon.cpp) are actively and consistently utilized by the + corresponding operator entry point and worker threads in `htp/*-ops.c`. + +### Tracing Instrumentation + +- All kernels must include trace events for performance profiling and timeline visualization in Perfetto + ([`hex-profile.h`](../../../ggml/src/ggml-hexagon/htp/hex-profile.h)). +- Surround compute sections with `htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)` and + `htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)`. +- Use specific event types for major phases: + - `HTP_TRACE_EVT_HVX_COMP`: Vector compute execution. + - `HTP_TRACE_EVT_DMA`: DMA transfer wait or poll cycles. + - `HTP_TRACE_EVT_FENCE`: Multi-device fence barrier synchronization. + - `HTP_TRACE_EVT_L2FLUSH`: L2 cache cleaning operations. +- Pass meaningful progress metrics (such as row index, chunk index, or token index) in the 16-bit `info` parameter. + +### Work Queue and Threading + +- Distribute parallel work across NPU worker threads using the thread pool work queue: + + ```c + work_queue_run(ctx->work_queue, worker_func, &op_ctx, n_threads); + ``` -Alternatively, users can choose to use standard llama.cpp/GGML layer-splitting mode to partition and split the model across -multiple Hexagon devices or virtual sessions (which behave like multiple GPUs from the offload and splitting perspective). +- Keep worker functions independent and re-entrant. Worker threads should only operate on their designated chunk of rows or elements. -Here is an example of running GPT-OSS-20B model on a Snapdragon device using 4 virtual sessions on a single NPU (physical index 0). +### Avoid Redundant Defensive NULL Checks +- Do not add defensive NULL checks or assertions for internal framework pointers or required graph operands and outputs. + Internal pointers include `ctx`, `octx`, local context structs like `*ctx`, `kparams`, and worker callback `data`. +- These pointers are architectural invariants during kernel execution and host-side graph preparation. + Graph compute receives allocated nodes with valid required `node->src[N]` and `node->data` pointers. +- Do not turn an invariant violation into an unsupported operation or missed fusion. + Checks such as `if (!octx || !octx->ctx)` clutter the code, obscure intent, and hide upstream errors. +- **Distinction**: `octx->src[N]` pointers *can* be NULL by design and must be checked when optional. + Examples include attention masks, optional bias or weights in fused kernels, and frequency factors. + +### Multiline Macro Formatting + +- Keep trailing backslashes in multiline `#define` macros cleanly aligned to a consistent column. +- Avoid trailing whitespace after macro backslashes. +- Use [`scripts/snapdragon/ggml-hexagon-align-macros.py`](../../../scripts/snapdragon/ggml-hexagon-align-macros.py) to inspect, diff, + or automatically align macro definitions across Hexagon kernel sources: + + ```bash + # Check for misaligned macros + python3 scripts/snapdragon/ggml-hexagon-align-macros.py ggml/src/ggml-hexagon/htp/ + + # Fix misaligned macros in-place + python3 scripts/snapdragon/ggml-hexagon-align-macros.py --fix ggml/src/ggml-hexagon/htp/ + ``` + +### Binary Inspection and Spill Analysis + +Use [`scripts/snapdragon/ggml-hexagon-inspect.py`](../../../scripts/snapdragon/ggml-hexagon-inspect.py) to audit Hexagon binaries for register +spills, unexpected float promotions, or disassembly: + +- Always verify that compute kernels have zero in-loop vector spills (`--spills --strict`) and no float promotions (`--promotions`). +- Avoid excessive loop unrolling (`#pragma unroll`), which increases register pressure and causes spills. + +```bash +# Check for vector and scalar register spills +python3 scripts/snapdragon/ggml-hexagon-inspect.py --spills --strict --func "^compute_" + +# Check for float promotions +python3 scripts/snapdragon/ggml-hexagon-inspect.py --promotions --func "^compute_" + +# Disassemble with annotated loops and spill markers +python3 scripts/snapdragon/ggml-hexagon-inspect.py --disasm compute_same_shape_div_f32 + +# Resolve crash addresses to function symbols and lines +python3 scripts/snapdragon/ggml-hexagon-inspect.py --addr2line 0x51a30 0x5ba54 ``` -~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --devices HTP0:0,HTP0:1,HTP0:2,HTP0:3 -- llama-cli --load-mode none -m /data/local/tmp/gguf/gpt-oss-20b-Q4_0.gguf -t 4 --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 -no-cnv -f surfing.txt -... -llama_model_loader: - type f32: 289 tensors -llama_model_loader: - type q4_0: 96 tensors -llama_model_loader: - type q8_0: 2 tensors -llama_model_loader: - type mxfp4: 72 tensors -... -load_tensors: offloaded 25/25 layers to GPU -load_tensors: CPU model buffer size = 1182.09 MiB -load_tensors: HTP0:1 model buffer size = 2512.58 MiB -load_tensors: HTP0:3 model buffer size = 2093.83 MiB -load_tensors: HTP0:0 model buffer size = 2931.34 MiB -load_tensors: HTP0:2 model buffer size = 2512.58 MiB -... -llama_context: n_ctx_per_seq (8192) < n_ctx_train (131072) -- the full capacity of the model will not be utilized -llama_context: CPU output buffer size = 0.77 MiB -llama_kv_cache_iswa: creating non-SWA KV cache, size = 8192 cells -llama_kv_cache: HTP0:1 KV buffer size = 25.50 MiB -llama_kv_cache: HTP0:3 KV buffer size = 25.50 MiB -llama_kv_cache: HTP0:0 KV buffer size = 25.50 MiB -llama_kv_cache: HTP0:2 KV buffer size = 25.50 MiB -llama_kv_cache: size = 102.00 MiB ( 8192 cells, 12 layers, 1/1 seqs), K (q8_0): 51.00 MiB, V (q8_0): 51.00 MiB -llama_kv_cache_iswa: creating SWA KV cache, size = 256 cells -llama_kv_cache: HTP0:1 KV buffer size = 0.80 MiB -llama_kv_cache: HTP0:3 KV buffer size = 0.53 MiB -llama_kv_cache: HTP0:0 KV buffer size = 1.06 MiB -llama_kv_cache: HTP0:2 KV buffer size = 0.80 MiB -llama_kv_cache: size = 3.19 MiB ( 256 cells, 12 layers, 1/1 seqs), K (q8_0): 1.59 MiB, V (q8_0): 1.59 MiB -llama_context: HTP0:0 compute buffer size = 16.06 MiB -llama_context: HTP0:1 compute buffer size = 16.06 MiB -llama_context: HTP0:2 compute buffer size = 16.06 MiB -llama_context: HTP0:3 compute buffer size = 16.06 MiB -llama_context: CPU compute buffer size = 98.19 MiB -... -llama_perf_context_print: prompt eval time = 3843.67 ms / 197 tokens ( 19.51 ms per token, 51.25 tokens per second) -llama_perf_context_print: eval time = 1686.13 ms / 31 runs ( 54.39 ms per token, 18.39 tokens per second) -llama_perf_context_print: total time = 6266.30 ms / 228 tokens -llama_perf_context_print: graphs reused = 30 -llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | -llama_memory_breakdown_print: | - HTP0:0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP0:1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP0:2 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP0:3 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - Host | 1476 = 1208 + 105 + 162 | + +## Multi-Device Partitioning (mdev) + +Multi-device (mdev) mode enables row-level tensor parallel execution across multiple physical NPU cores or virtual NPU +sessions. + +### 128-Byte Cache Line Alignment + +- Shared tensor buffers reside in DDR (RPCMEM) with a 128-byte cache line granularity + (`HEX_L2_LINE_SIZE` = 128 bytes, `HTP_TENSOR_MDEV_LINE_SIZE`). +- **Rule**: Multi-device work partitions must align destination write regions to 128-byte cache line boundaries so distinct + devices never share or overwrite the same cache line. + +### Partitioning Helpers in `htp-tensor.h` + +Common partitioning logic is factored into reusable inline helpers in +[`htp-tensor.h`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h): + +1. [`htp_tensor_mdev_rows_per_chunk`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L67): + Determines the minimum number of rows per chunk so that the chunk byte size is a multiple of 128 bytes: + + ``` + rows_per_chunk = 128 / hex_gcd_u32(row_size, 128) + ``` + + If row stride `nb[1]` is already a multiple of 128 bytes, `rows_per_chunk = 1`. + Returns `false` if the tensor cannot be safely row-partitioned (such as unaligned base pointer, permuted layout, + or non-128-byte aligned outer strides). + +2. [`htp_tensor_mdev_partition`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L94): + Calculates the per-device work range `struct htp_tensor_mdev_range { uint32_t start; uint32_t count; }` given + `total_units`, `units_per_chunk`, `mdev_idx`, `mdev_count`, and the precomputed `mdev_count_div`. + Handles chunk distribution across devices, assigns remainder units to the last device, and automatically triggers + single-device fallback when partitioning is unsafe. + +### Row-Partitioned Operators + +For row-wise operators +(such as activations in [`act-ops.c`](../../../ggml/src/ggml-hexagon/htp/act-ops.c), +binary ops in [`binary-ops.c`](../../../ggml/src/ggml-hexagon/htp/binary-ops.c), +unary ops in [`unary-ops.c`](../../../ggml/src/ggml-hexagon/htp/unary-ops.c), and +sameshape copies in [`cpy-ops.c`](../../../ggml/src/ggml-hexagon/htp/cpy-ops.c)): + +```c +const uint32_t total_rows = ne01 * ne02 * ne03; +const size_t dst_row_size = dst->ne[0] * elem_size; + +uint32_t row_start = 0; +uint32_t nrows = total_rows; + +if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, elem_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; +} + +if (nrows == 0) { + return HTP_STATUS_OK; +} ``` + +### Element-Partitioned Operators + +For flat element-wise operations (such as reshape copies in +[`cpy-ops.c`](../../../ggml/src/ggml-hexagon/htp/cpy-ops.c)): +- Partition total linear elements N = ne0 * ne1 * ne2 * ne3 in 128-byte cache line chunks (`elems_per_line = (elem_size == 4) ? 32 : 64`). +- Requires strict 1D contiguity: + [`htp_tensor_is_contiguous(dst, elem_size)`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L28) + and 128-byte aligned destination pointer + [`htp_tensor_mdev_data_aligned(dst)`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L47). +- If contiguous and aligned, pass `elems_per_line` to + [`htp_tensor_mdev_partition`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L94); + otherwise pass 0 to trigger Device 0 fallback. + +### Single-Device Fallback (Device 0) + +- Fallback to Device 0 (`mdev.idx == 0`) when partitioning would cause cache line tearing or when work cannot be evenly distributed. +- Triggers: + 1. Destination tensor cannot be safely partitioned (`rows_per_chunk == 0` or non-contiguous/unaligned buffer). + 2. Total aligned chunks < `mdev_count`. +- Device 0 processes the entire tensor `[0, total_units)`. +- Devices 1 ... N-1 receive `count = 0` and return `HTP_STATUS_OK` immediately. + +### Flatten Outer Dimensions Globally + +- **Never partition solely on `ne01` (dimension 1).** +- Partitioning only on `ne01` repeats the device boundary across every 2D slice (`ne02`, `ne03`). If each 2D slice is small, + false sharing occurs repeatedly throughout the tensor. +- Always flatten outer dimensions globally: `total_rows = ne01 * ne02 * ne03` and partition once across the combined row space. + +### Stateless Starting Coordinates + +- Do not use incremental state variables across slices that assume the thread or device starts at index 0. +- Precompute starting multidimensional coordinates at `r = row_start` (or `e = elem_start`) once using `fastdiv`. +- In inner loops, step base pointers directly (`ptr += stride`) or reset/wrap coordinates explicitly (`if (++i01 == ne01) { ... }`). + +### Clean Range Encapsulation + +- Initialize single-device default ranges at declaration: + + ```c + uint32_t row_start = 0; + uint32_t nrows = total_rows; + ``` + +- Encapsulate all multi-device logic inside `if (octx->ctx->mdev.count > 1)`. If the block is omitted or compiled out, + the operator runs standard single-device execution untouched. +- Do not propagate `mdev_` prefixes to worker functions or context structs. Worker threads are device-agnostic and + should only receive standard range parameters (`ctx.row_start`, `ctx.nrows`). +- In worker threads, calculate row intervals using standard arithmetic: + + ```c + const uint32_t ir0 = ctx->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, ctx->row_start + ctx->nrows); + ``` + + In single-device mode (`row_start == 0`), this naturally simplifies to `dr * ith` and `MIN(ir0 + dr, ctx->nrows)` with zero overhead. + +## Multi-Device Synchronization + +Multi-device execution synchronizes worker sessions across devices using explicit barriers and tensor cache flushing. + +### Synchronization Fence Protocol + +Multi-device execution synchronizes worker sessions through atomic fence slots and barriers defined in +[`htp-fence.h`](../../../ggml/src/ggml-hexagon/htp/htp-fence.h): + +``` +[NPU Session 0] [NPU Session 1] + | | + (Input Prep) (Input Prep) + | | + Pre-Op Barrier ----------------------------- Pre-Op Barrier + (mdev_sync_fence) (mdev_sync_fence) + | | + Kernel Execution Kernel Execution + (Output Slice 0) (Output Slice 1) + | | + Tensor Cache Flush Tensor Cache Flush + (htp_tensor_flush_all) (htp_tensor_flush_all) + | | + Post-Op/Batch Barrier ---------------------- Post-Op/Batch Barrier + (htp_mdev_group_barrier) (htp_mdev_group_barrier) + | | + Return Response to Host Return Response to Host +``` + +### Atomic Fence Slots and Cache Invalidation + +- Fence synchronization operates on dedicated RPCMEM shared memory mapped across all participating sessions (`ctx->mdev.fence_base`). +- Each device owns a dedicated 128-byte cache-line aligned fence slot: + + ```c + atomic_uint * my_fence = htp_mdev_fence_slot(fence_base, mdev_idx); + ``` + +- **Writing to fence ([`htp_fence_write`](../../../ggml/src/ggml-hexagon/htp/htp-fence.h#L18))**: + Stores `seq` and `status`, issues a `syncht` thread synchronization barrier, and flushes/invalidates the line + using `Q6_dccleaninva_A(fence)`. +- **Reading from peer fence ([`htp_fence_read`](../../../ggml/src/ggml-hexagon/htp/htp-fence.h#L26))**: + Executes `Q6_dccleaninva_A(fence)` and `syncht` before reading atomic values to ensure fresh data from DDR. + +### Deterministic Monotonic Sequence Numbers + +- Barrier fences use monotonically increasing sequence numbers: + + ```c + const uint32_t seq = ++ctx->mdev.fence_seq; + ``` + +- Comparing sequence numbers with signed arithmetic `(int32_t)(peer_seq - seq) >= 0` prevents race conditions or + misaligned barrier arrivals across iterations. +- If any peer reports an error status (`peer_status > HTP_STATUS_OK`), the barrier propagates the error and unblocks immediately. + +### Tensor Cache Flush and Pipeline Completion + +- In the kernel, ensure all pushed DMA operations have been popped in strict FIFO order to drain the queue. +- Use [`htp_tensor_flush_all()`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h) to flush specific dirty tensors back to DDR: + - [`htp_tensor_flush_all()`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h) flushes only modified tensor address ranges, + ensuring peer devices and the host CPU observe consistent data in DDR. +- Never signal completion before all DMA transfers are drained and dirty tensor flushes have completed. + diff --git a/docs/build-profiling.md b/docs/build-profiling.md new file mode 100644 index 000000000000..839e7cca4cd5 --- /dev/null +++ b/docs/build-profiling.md @@ -0,0 +1,122 @@ +## Build profiling +This page is a working document for analyzing the current build and try to +identify ways to improve the build time. + +### Requirements +The profiling script requires clang to be used as the compiler tool chain and +also requires that ClangBuildAnalyzer is installed. + +Mac: +```console +brew install clang-build-analyzer +``` + +Linux: +```console +git clone https://github.com/aras-p/ClangBuildAnalyzer.git +cd ClangBuildAnalyzer +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build -j$(nproc) +sudo cp build/ClangBuildAnalyzer /usr/local/bin/ +``` + +Windows: install LLVM/clang and Ninja (e.g. via the +[LLVM releases page](https://github.com/llvm/llvm-project/releases) and +`winget install Ninja-build.Ninja`), then build ClangBuildAnalyzer the same +way as on Linux: +```console +git clone https://github.com/aras-p/ClangBuildAnalyzer.git +cd ClangBuildAnalyzer +cmake -B build -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release +cmake --build build --config Release +``` +Then add `ClangBuildAnalyzer\build` to `PATH`. + +### Usage +Mac/Linux: +```console +$ ./scripts/build-profile.sh +``` + +Windows: +```console +> .\scripts\build-profile.ps1 +``` + +Both accept `--full`/`-Full` (include Server, Tools, and Tests) and a jobs +override (`-jN` / `-Jobs N`). + +Note: on Windows, `cmake` defaults to the Visual Studio generator, which +ignores `CMAKE_C_COMPILER`/`CMAKE_CXX_COMPILER` and silently falls back to +MSVC. `build-profile.ps1` passes `-G Ninja` so clang is actually used, this +is required on ARM64. + +### Linux (Ubuntu 24.04) + +Environment: +- Clang: 18.1.3 (Ubuntu clang version 18.1.3 (1ubuntu1)) +- libstdc++: GCC 13.3.0 (Ubuntu 13.3.0-6ubuntu2~24.04.1) +- Target: x86_64-pc-linux-gnu + +```console ++------------------------+-----+------------+------------+------------+ +| Build | TUs | Frontend | Backend | Total | ++------------------------+-----+------------+------------+------------+ +| Minimal, master | 249 | 468.2 s | 270.3 s | 738.5 s | +| Minimal, with PCH | 253 | 177.1 s | 265.8 s | 442.9 s | +| Full, master | 396 | 811.0 s | 692.2 s | 1,503.2 s | +| Full, with PCH | 405 | 380.0 s | 664.7 s | 1,044.7 s | +| Full, with PCH + UB | 264 | 357.7 s | 635.7 s | 993.4 s | ++------------------------+-----+------------+------------+------------+ + +PCH = precompiled header. +Full = includes building Server, Tools, and Tests. +UB = unity build for models +``` +Note that the number of translation units (TUs) increases when using precompiled +headers — each PCH target adds one extra TU for the precompilation step itself. + +### Mac (Apple M3) + +Environment: +- Clang: Apple clang version 17.0.0 (clang-1700.3.19.1) +- libc++: ships with Apple clang 17.0.0 (Xcode toolchain) +- Target: arm64-apple-macosx15.6 + +```console ++------------------------+-----+------------+------------+------------+ +| Build | TUs | Frontend | Backend | Total | ++------------------------+-----+------------+------------+------------+ +| Minimal, master | 256 | 154.5 s | 94.8 s | 249.3 s | +| Minimal, with PCH | 261 | 65.9 s | 90.0 s | 155.9 s | +| Full, master | 407 | 265.7 s | 209.7 s | 475.4 s | +| Full, with PCH | 414 | 154.6 s | 197.5 s | 352.1 s | +| Full, with PCH + UB | 274 | 143.0 s | 192.2 s | 335.2 s | ++------------------------+-----+------------+------------+------------+ + +PCH = precompiled header. +Full = includes building Server, Tools, and Tests. +UB = unity build for models +``` + +### Windows (ARM64) + +Environment: +- Clang: clang version 22.1.8 (LLVM, `C:\Program Files\LLVM`) +- STL: MSVC STL (Visual Studio 2022 Build Tools 14.44.35207) +- Target: aarch64-pc-windows-msvc + +```console ++------------------------+-----+------------+------------+------------+ +| Build | TUs | Frontend | Backend | Total | ++------------------------+-----+------------+------------+------------+ +| Minimal, master | 249 | 159.4 s | 82.2 s | 241.6 s | +| Full, master | 373 | 337.2 s | 167.4 s | 504.6 s | +| Minimal, with PCH + UB | 113 | 62.3 s | 82.4 s | 144.7 s | +| Full, with PCH + UB | 240 | 233.0 s | 185.1 s | 418.1 s | ++------------------------+-----+------------+------------+------------+ + +PCH = precompiled header. +Full = includes building Server, Tools, and Tests. +UB = unity build for models +``` diff --git a/docs/build-s390x.md b/docs/build-s390x.md index 4568d5010f6c..005dd2983459 100644 --- a/docs/build-s390x.md +++ b/docs/build-s390x.md @@ -243,6 +243,7 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl | FP32 | ✅ | ✅ | ❓ | | FP16 | ✅ | ✅ | ❓ | | BF16 | ✅ | ✅ | ❓ | +| Q1_0 | ✅ | ❓ | ❓ | | Q4_0 | ✅ | ❓ | ❓ | | Q4_1 | ✅ | ❓ | ❓ | | MXFP4 | ✅ | ❓ | ❓ | @@ -272,4 +273,4 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl - 🚫 - acceleration unavailable, will still run using scalar implementation - ❓ - acceleration unknown, please contribute if you can test it yourself -Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Feb 15, 2026. +Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Sep 8, 2026. diff --git a/docs/build.md b/docs/build.md index f794d490b097..70fc17af2402 100644 --- a/docs/build.md +++ b/docs/build.md @@ -300,7 +300,8 @@ The following compilation options are also available to tweak performance: |-------------------------------|------------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | GGML_CUDA_FORCE_MMQ | Boolean | false | Force the use of custom matrix multiplication kernels for quantized models instead of FP16 cuBLAS even if there is no int8 tensor core implementation available (affects V100, CDNA and RDNA3+). MMQ kernels are enabled by default on GPUs with int8 tensor core support. With MMQ force enabled, speed for large batch sizes will be worse but VRAM consumption will be lower. | | GGML_CUDA_FORCE_CUBLAS | Boolean | false | Force the use of FP16 cuBLAS instead of custom matrix multiplication kernels for quantized models. There may be issues with numerical overflows (except for V100, CDNA and RDNA4 which use FP32 compute type by default) and memory use will be higher. Prompt processing may become faster on recent datacenter GPUs (the custom kernels were tuned primarily for RTX 3000/4000). | -| GGML_CUDA_FA_ALL_QUANTS | Boolean | false | Compile support for all KV cache quantization type (combinations) for the FlashAttention CUDA kernels. More fine-grained control over KV cache size but compilation takes much longer. | +| GGML_CUDA_FA_QUANTS | `all` or `type_K-type_V` list | q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16 | Select which K/V type combinations to compile the FlashAttention CUDA kernels for. `all` compiles every combination, but compilation takes much longer. Otherwise a `;`-separated list of `type_K-type_V` pairs; f16-f16 is always compiled. Combinations that were not compiled fall back to f16-f16 kernel with a warning. Legal types: f16, bf16, q4_0, q4_1, q5_0, q5_1, q8_0. | +| GGML_CUDA_FA_ALL_QUANTS | Boolean | false | Deprecated alias for `GGML_CUDA_FA_QUANTS=all`. | ## MUSA @@ -805,7 +806,7 @@ To read documentation for how to build on Android, [click here](./android.md) ## WebGPU -The WebGPU backend relies on [Dawn](https://dawn.googlesource.com/dawn). Follow the instructions [here](https://dawn.googlesource.com/dawn/+/refs/heads/main/docs/quickstart-cmake.md) to install Dawn locally so that llama.cpp can find it using CMake. The current implementation is up-to-date with Dawn commit `18eb229`. +The WebGPU backend relies on [Dawn](https://dawn.googlesource.com/dawn). Follow the instructions [here](https://dawn.googlesource.com/dawn/+/refs/heads/main/docs/quickstart-cmake.md) to install Dawn locally so that llama.cpp can find it using CMake. The current implementation is up-to-date with Dawn commit `94c3c9c`. In the llama.cpp directory, build with CMake: diff --git a/docs/dev/2026-09-24-upstream-sync.md b/docs/dev/2026-09-24-upstream-sync.md new file mode 100644 index 000000000000..ab8b03f8ec40 --- /dev/null +++ b/docs/dev/2026-09-24-upstream-sync.md @@ -0,0 +1,167 @@ +# Upstream sync 2026-09-24 + +Merge of `upstream/master` (ggml-org/llama.cpp @ `bd4f514`) into the fork (origin/master @ `26e5f58`). + +- Merge base: `427291b5b` (metal : add remaining fa-vec tunings for M3 (#28396)) +- Upstream commits merged: 335 +- Fork commits since base: 1570 +- Conflicted files: 32 (~90 hunks) +- Branch: `claude/hello-ky3ivj` (not master; review before merging) + +Protocol: same as the earlier syncs. No blind `--theirs` on any customized file. Where upstream added a mechanism that does the same job as fork code, both were compared and one kept (reasons below). Fork lines that the merge dropped were audited afterwards. + +Trivial "keep both sides" hunks (independent additions next to each other) are not listed. Everything that needed a decision is. + +## Decisions + +### common/common.h - `hostname` -> `hostnames` +Upstream replaced `std::string hostname` with `std::vector<std::string> hostnames` (multi-address bind). The conflict only came from our paged-KV / semantic-index params sitting right above it. Kept all fork params, took upstream's `hostnames`. No fork code reads `params.hostname`. + +### common/chat.cpp - parsers moved to `common/parsers/` (upstream #27764) +Upstream split the per-model chat parsers out of `chat.cpp` into `common/parsers/*.cpp`. That is the whole 2500-line conflict. Our side had two small fixes inside the moved functions. Took upstream's `chat.cpp` and ported both fixes to the new files: +- `common/parsers/lfm2.cpp`: LFM2.5 optional opening `<think>` (fork `bf90040d7`). Upstream still has the strict `THINK_START + ...` form, so the reasoning-bleed bug would come back without the port. +- `common/parsers/deepseek.cpp`: DeepSeek V4.1 DSML tag names with a leading space (fork `6274e38a7`). Upstream has no V4.1 detection. + +### common/speculative.cpp - draft split mode (upstream #28390 vs fork MAD-LAB) +**Overlap: both sides solved the same bug.** A draft model inherits the target's `-sm tensor` and gets its own Meta device wrapper. +- Upstream: force `LLAMA_SPLIT_MODE_LAYER` only when `-devd` names exactly one device. +- Fork: force `LAYER` for every separate draft model. + +**Kept: fork.** The fork's version covers upstream's case (one draft device) and also covers multi-device and inherited-device drafts. In this fork the draft context co-schedules on the target's Meta backend (see the MAD-LAB note in llama_context backend init), and two Meta devices in one scheduler abort in `ggml_backend_meta_buffer_simple_tensor()`. Upstream's narrower check would bring that abort back for `-devd A,B` or a draft that inherits the target devices. The fork also avoids a per-layer AllReduce on the latency-critical draft path. Upstream's comment and code were dropped. + +### common/speculative.cpp - M-RoPE pinned-image skip in DFlash inject +Upstream added a skip for image rows pinned to one position. Kept it and put it **before** the fork's `DFlash inject` stats and `WP_DSPARK_DEBUG` census, so skipped rows are not logged as injected. Also took upstream's "frist" -> "first" typo fix next to our hash-trace block. + +### CUDA/HIP FlashAttention instance selection (upstream #28079 vs fork turbo lists) +**Overlap.** Upstream replaced `GGML_CUDA_FA_ALL_QUANTS` with `GGML_CUDA_FA_QUANTS`: a list of `K-V` pairs, per-pair `GGML_CUDA_FA_<K>_<V>` defines, and a shared `ggml_cuda_fattn_vec_instances()` in `ggml/cmake/common.cmake` used by CUDA, HIP and MUSA. When a pair was not compiled, the dispatcher now falls back to f16 conversion (with a one-time warning) instead of aborting. It also dropped the `K->type != V->type -> NONE` gate. + +The fork had three pieces that did the same job: turbo instance files hard-coded into the CUDA and HIP `else()` lists, turbo `FATTN_VEC_CASES_ALL_D` rows placed before `GGML_ABORT`, and a widened `K != V` gate that let mixed turbo/q8_0 pairs through. + +**Kept: upstream's mechanism, extended for turbo.** +- `ggml/cmake/common.cmake`: new `FA_TURBO_COMBINATIONS` (the same 15 pairs the fork compiled before). They are always compiled, whatever `GGML_CUDA_FA_QUANTS` is set to, and they get `GGML_CUDA_FA_TURBO*_*=1` defines. They are not added to `FA_TYPES`, because `GGML_CUDA_FA_QUANTS=all` crosses every type with every other and there are no turbo x q4_0 (etc.) instance files. That would be a configure-time FATAL_ERROR. +- `fattn.cu`: the turbo rows moved into `ggml_cuda_get_fattn_vec_case()` with upstream's short type names. The fork's mixed-type gate was dropped because upstream has no gate left to widen. +- CUDA/HIP `CMakeLists.txt`: took upstream (the function call). + +Why: upstream's version can be configured and fails soft. A turbo pair we did not compile (for example turbo3-f16) now runs via f16 conversion instead of aborting. MUSA also gets the turbo instances now. Before this it referenced them without compiling them. The fork's `GGML_CUDA_FA_ALL_QUANTS=ON` still works; upstream maps it to `all` with a deprecation warning. + +### ggml/src/ggml-cuda/vendors/hip.h - host-alloc aliases +**Overlap.** Upstream added four of the six `cudaHostAlloc*` -> `hipHostMalloc*` aliases that the fork already had for the AllReduce pipeline. Kept the fork's block because it is a superset (it also has `cudaHostAllocDefault` and `cudaDevAttrCanMapHostMemory`) and dropped upstream's duplicate defines. + +### src/CMakeLists.txt - unity build (upstream #28091) +Upstream moved the core sources into `LLAMA_CORE_SOURCES` and turned on `UNITY_BUILD` for the rest (the `models/*.cpp` files, in batches of 16). Core sources are excluded. Added the fork's core files (`llama-tp-lockstep`, `pipeline/pipe-tp-msg`, `llama-kv-cache-paged`, `llama-ml8-registry`, `llama-pipeline`, `llama-pipeline-gguf`, `llama-weight-pager`) to `LLAMA_CORE_SOURCES`. Also excluded `weight-pager/*.cpp` and `memory-tier/*.cpp` from the unity build. Those are large, standalone subsystems that were never written for single-TU inclusion. The fork's own `models/*.cpp` now take part in the unity build, the same as upstream's (see the build check at the end). + +### tools/server/server-models.cpp - router scheduler (upstream #29217, #28539, #28530) +Upstream rewrote `server_lru_sched`. Every load now goes through the queue (`join` -> `try_claim` -> `load`). `tick()` evicts idle models while queued requests outnumber free slots, and replaces `on_model_idle` / `mark_slot_pending`. `pick_victim` lost its `exclude` argument and skips any model a queued request wants. Requests are no longer admitted into a stopping model. The fork had not changed `ensure_model_ready`, so upstream's version came in unchanged. The fork-specific pieces were merged one at a time: + +- **`pick_victim` guards:** kept the fork's **pinned** hard hold (upstream has no pinning) and took upstream's "stopping or queued" filter, which covers the fork's old stopping guard. `tick()` also goes through `pick_victim`, so pinned models are safe from queue-driven eviction too. +- **`unload_lru()`**, overlapping fix: **kept fork** (with the new one-argument `pick_victim`). The fork marks the victim as stopping under the same lock that picked it, returns early when nothing can be evicted, and waits with the find-based `wait()`. Upstream's version waits on `mapping[name]`, which default-constructs a stray entry if the model was erased mid-wait. The fork's version is the more race-safe of the two. +- **Capacity re-check in `load()`**, overlapping: upstream throws "model limit reached" and relies on the queue to prevent the race. The fork evicts and waits (30 s cap). **Kept the fork's loop and added upstream's `opts.mode == SERVER_CHILD_MODE_NORMAL` exemption** so download workers do not take `models_max` slots. On the request path the queue guarantees capacity before `load()` runs, so the loop does nothing there. It still matters for loads that bypass the queue (the `/models/load` API, startup preload, reload), where a 500 is worse than a short wait. +- **Proxy cleanup:** kept the fork's chained cleanup and `last_used` re-stamp at response end (so the idle sweeper does not unload mid-stream). Swapped the removed `sched->on_model_idle()` for upstream's `sched->tick(lk)` under the same lock. +- **`update_status`:** kept `reconcile_gpu_reservation_locked()` and added upstream's `sched->tick(lk)` after it. +- **Preset load:** kept the fork's preset-mtime tracking and took upstream's `SRV_INF` -> `SRV_TRC` log level change. + +### src/llama-context.{h,cpp} - graph result cache (upstream #28549 vs fork `WP_GRAPH_RESULT_SLOTS`) +**Overlap: both sides cache more than one graph result so alternating graph shapes (MTP draft vs verify) stop fighting over one slot.** +- Upstream: `gf_res_prev` became `std::array<..., 2>`, picked by `n_outputs > 0`, with a `gf_res_prev_active` pointer. Both arenas share the context's single scheduler, so a ggml graph is only reused when its arena is the one currently allocated. The gain is that the CUDA backend sees stable graph keys per arena and can reuse captured CUDA graphs. Default on. +- Fork: `gf_slots` keyed by `(gtype, n_tokens)`, and **each slot owns its own scheduler**. A hit therefore skips the ggml graph build and allocation, not only the CUDA re-capture. Slot 0 aliases the single `gf_res_prev`. Opt-in via `WP_GRAPH_RESULT_SLOTS=N` (default 1 = legacy). + +**Kept: fork.** Removed upstream's 2-arena array, `get_gf_res_prev()` and `gf_res_prev_active`, including the parts that auto-merged cleanly (`sched_reserve`, `process_ubatch`, `opt_epoch_iter`). +Why: +1. The two designs cannot both own `gf_res_prev`. Many fork paths treat it as the single slot-0 result tied to the context `sched`: the TP-overlap rolling loop (`gf_res_prev` / `gf_res_overlap`), the Qwen4exp staged layer-cut path, `output_project`'s max-nodes sizing, and `wp_reset_graph_results()`. Lazily allocating a second arena under them would break the "one sched sees one topology" invariant that the fork's `c7b8001ff` post-mortem (`HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION`) is about. +2. The fork's design is a functional superset. With `WP_GRAPH_RESULT_SLOTS>=2` the verify and draft shapes land in separate slots, each with its own allocator, so there is real graph reuse instead of only CUDA-graph reuse. +3. Upstream's gain depends on CUDA graphs. On this fork's ROCm fleet they only apply with `GGML_HIP_GRAPHS`, which is off by default. +4. The `gf_res_prev_active` guard only exists because upstream's two arenas share one scheduler. The fork's per-slot scheduler already keeps that invariant, and `opt_epoch_iter` still calls `res->reset()`, which makes the next `can_reuse()` fail. + +**Needs your review:** if you run NVIDIA with CUDA graphs and default `WP_GRAPH_RESULT_SLOTS=1`, this loses upstream's MTP CUDA-graph gain. Setting `WP_GRAPH_RESULT_SLOTS=2` is the fork-native equivalent. + +### src/llama-graph.cpp - HC-arch list +Upstream added `LLM_ARCH_MAPLE` to the DSv4-style hyper-connection arch check and the fork had added `LLM_ARCH_DEEPSEEK41`. Kept both. + +### src/llama-model.cpp - meta-split tensor config +Upstream mirrors every tensor for `LLM_ARCH_HRM_TEXT` (aliased cache slots). The fork added NextN/MTP and ml8-sidecar split rules at the same spot. Put upstream's arch check **first** so an HRM model never reaches the fork's per-tensor rules. + +### src/models/kimi-k3.cpp - GDN l2 norm +Both sides made the same fix (`ggml_l2_norm` -> `build_gdn_l2_norm`). Took upstream's text (`eps_norm` naming). + +### src/models/qwen4exp.cpp - HC grouped-norm fusion (upstream #28896 vs fork `WP_QWEN4EXP_FUSE_HC_NORM`) +**Overlap.** Both sides wanted RMS_NORM and MUL next to each other so the backend can fuse them. +- Fork: opt-in env flag that reshapes the `[hc_dim]` gamma into a 3-D view inside the graph. +- Upstream: loads the gamma as `[n_embd, hc]` (`TENSOR_ALLOW_RESHAPE`) and always multiplies before the reshape. Same math, no env flag, no graph reshape. + +**Kept: upstream.** It gives the fused form by default with fewer nodes. It also has to win, because the tensors now load as `[n_embd, hc]`. The fork's fallback branch (reshape to `[hc_dim, nt]`, then multiply) would no longer broadcast. `WP_QWEN4EXP_FUSE_HC_NORM` is gone (it is now always effectively on). +**Extra port:** the fork's MTP/NextN block (not in upstream) created `hc_attn_norm` / `hc_ffn_norm` as `{ hc_dim }`. Changed both to `{ n_embd, hc }` with `TENSOR_ALLOW_RESHAPE`, because they go through the same `build_hc_mix`. Without that, the MTP head would assert on the broadcast. + +### src/models/qwen4exp.cpp - sparse FA (upstream #28770) +Upstream turned on the sparse attention hint (`top_k->ne[0]`). The fork added the TurboQuant Q forward-rotation just before that call. Kept both. The top-k value is only an `n_kv_max` hint that the CUDA MMA-f16 kernel uses. Turbo K goes through the vec kernel, which ignores it, and the mask stays authoritative, so correctness does not depend on it. + +### src/models/dflash.cpp - Gemma4 DSpark backbone (upstream #29226) and build_arch_graph move (#28934) +- Layer tensors: kept the fork's optional softplus attention gate (Laguna-style drafter) and added upstream's optional post-norms, `out_scale` and `rope_freqs`. +- Upstream moved `build_arch_graph()` below the graph template specialisations. Dropped the fork's copy at the old spot (the moved copy is identical) and kept the fork's `build_dflash_encode()` helper there. +- Embedding scale: upstream added `f_embedding_scale` after `get_rows`. The fork's three-way embedding source (own table / borrowed target table / DSpark services mode, where the driver passes in gathered target rows) was kept. The scale is applied **after** all three, because the services-mode rows are the target's raw table rows, the same thing upstream scales. + +### ggml-cuda/mmq.{cu,cuh} - MMQ tile sizing (upstream #28552 vs fork `WP_EXPERT_MM_PIN`) +**Overlap: both change the column count that MMQ picks its J tile against.** +- Upstream: new `mmq_args::ncols_opt`. On RDNA3/RDNA4 the routed-MoE path sizes tiles against the average tokens per expert (`ne12*n_expert_used/ne02`) instead of `ne12`. Everywhere else it is the same as before. +- Fork: when `force_mm_id` is set (the pinned expert mul_mat), tiles are sized against a fixed reference width (`MMQ_FORCE_MM_REFERENCE_TOKENS`, override `GGML_MMQ_PIN_REF_TOKENS`) so results do not depend on batch width. + +**Kept: both, with the fork taking priority.** `ncols_tile = force_mm_id && pin_ref_tokens > 0 ? pin_ref_tokens : args.ncols_opt`. The pin is a correctness contract (width-invariant expert outputs, which the wp-expert-worker path relies on). Upstream's gain is performance and still applies to every non-pinned mul_mat. `ncols_opt` goes right after `ncols_max` in the struct, before the fork's defaulted `force_mm_id` / `expert_ptrs`, so the positional brace-init in `mmq.cu` fills the correct field. +Also kept the fork's gfx80x (GCN4 and earlier) decode-only MMQ rule and took upstream's wider Vega/gfx909/gfx90c comment. + +### ggml-cuda/ggml-cuda.cu - BF16 cuBLAS fallback (upstream #28846 vs fork `4019c9a7b`) +**Overlap.** Both fall back from BF16 to F32 cuBLAS compute on GPUs without fast BF16. +- Fork: always F32 when `!bf16_mma_hardware_available(cc)`. +- Upstream: F32 only above a batch threshold (AMD `ne11 > 32`, NVIDIA `> 8` from Volta, else `> 128`) when `!fast_bf16_hardware_available(cc)`. Small batches keep BF16 compute. + +**Kept: upstream.** It was benchmarked and reviewed upstream (J. Gäßler co-author). It covers the fork's case for prefill-sized batches and avoids a conversion pass for small (decode) batches. Both use the same hardware sets on AMD/NVIDIA and both fall back to F32, not F16, so exactness is the same. Also kept the fork's FP8_B128 AITER unpack at the top of `ggml_cuda_mul_mat_cublas`. +**Needs your review:** on RDNA2/GCN with BF16 weights at batch <= 32, cuBLAS/hipBLAS now runs BF16 compute again instead of F32. If your earlier DS4 measurements showed this was slow at decode sizes, restore the fork's unconditional arm. + +### ggml-cuda/allreduce.cu - internal AllReduce on ROCm (upstream #27825 vs fork HIP port) +**Overlap: both sides ported the internal two-GPU AllReduce to HIP.** +- Upstream: the minimum. Drop the `GGML_USE_HIP` exclusion, use `__builtin_amdgcn_s_sleep(4)` in the spin, and change the Volta comment. +- Fork: a full port. `hipHostMalloc(Portable|Mapped|Coherent)` fine-grained staging so device stores reach host memory and not L2; `__hip_atomic_*` at system scope for the arrival handshake; a `cudaDevAttrCanMapHostMemory` capability check with fallback; a wall-clock spin watchdog; plus the duplex transport, codecs and diagnostics built on top. + +**Kept: fork** in all 9 hunks. It is a strict superset of upstream's change (same s_sleep idea, wrapped in `ggml_cuda_ar_spin_pause()`), and the coherence and atomic-scope pieces are what make it correct on multi-GPU PCIe/TB3 ROCm setups. Upstream's version leaves device stores to coarse-grained pinned memory with volatile accesses. Upstream's non-conflicting `CUDA_CHECK()` wrapping of teardown calls merged in cleanly and was kept. The MUSA stub keeps the fork's comment (it correctly says HIP *is* supported). + +### ggml-vulkan - upstream file split (#28732) plus 30 upstream Vulkan commits +**The largest piece of this sync.** Upstream moved ~5,400 lines out of `ggml-vulkan.cpp` into `ggml-vulkan-types.h`, `ggml-vulkan-push-constants.h`, `ggml-vulkan-common.h`, `ggml-vulkan-buffers.cpp` and `ggml-vulkan-debug.cpp`. Many functions went from `static` to shared (declared in `common.h`), and `ggml_vk_dispatch_pipeline` became an `inline` template in `common.h`. The fork had ~2,800 changed lines spread across those regions. A hunk-by-hunk text merge gave 90 diff3 hunks, most of which were "fork code sitting in a region upstream deleted", so resolving them in place would have silently dropped the moved-out copies. + +**Method:** started from upstream's split files and replayed the fork's own patch (`git diff 427291b5b HEAD -- ggml-vulkan.cpp`, 141 hunks). Each hunk was tried against `ggml-vulkan.cpp` and then each new file: 93 applied with exact context, 10 with fuzz 1 (each placement checked by hand), and 38 were ported by hand. Afterwards a line-level audit confirmed every fork-added line exists somewhere in the merged tree, except the lines replaced on purpose below. + +Hand-port decisions worth knowing about: +- **Queue quiescence epochs** (`WP_VK_HOST_READ_FASTPATH`): the atomics and helpers moved to `vk_queue_handle` in `types.h`. The `submit_epoch` bump goes into both `submit()` bodies, which are now out-of-line in `.cpp` and also carry upstream's NVIDIA `device_submit_mutex` workaround (#28830). The bump sits after the device lock, as before. +- **Graph plans** (`ggml_backend_vk_graph_plan`, `recording_plan`): the plan struct is in `types.h`. The descriptor pool/set accessors moved to `common.h` as `inline`, because upstream's `ggml_vk_dispatch_pipeline` template now lives there and has to see them. The fork's `dispatch_pipeline` (takes `vk_subbuffer`, records descriptor updates for plans) replaces upstream's copy in `common.h`. +- `ggml_vk_ctx_begin(..., bool one_time = true)`: the default moved to the `common.h` declaration. +- `vk_node_ts_enabled` is a shared global next to upstream's other logger flags in `ggml-vulkan-debug.cpp` (extern in `common.h`). `vk_want_timestamps()` is `inline` in `common.h`. +- Forward declarations the fork added for `ggml_vk_buffer_from_host_ptr` / `ggml_vk_is_empty` were **dropped**. Upstream now has both as shared functions, and a `static` redeclaration would not compile. +- **MUL_MAT pin (`GGML_VK_PIN_REF_TOKENS`, `force_mm`) vs upstream's pipeline maps (#25773):** upstream replaced `ggml_vk_guess_matmul_pipeline(ctx, mmp, ..., type, type)` with `..._map(ctx, *mmp_map, ...)`. The fork's `pipeline_n` override now goes into the map-based calls for both `mul_mat_q_f16` and `mul_mat_id_q_f16`, so the pinned tile choice is unchanged. +- **`ggml_vk_mul_mat_vec_q_f16` signature:** upstream added `bool swap_inputs = false` (#28457, one-row B^T*A trick) and the fork had added `bool pin = false` in the same position. The merged signature is `(..., bool swap_inputs = false, bool pin = false)`, and the fork's only `pin` caller (`ggml_vk_mul_mat_pinned_vec`) now passes `false, true`. Without this, the pinned path would have silently turned on input swapping. +- Upstream's new swap-inputs branch tests `dst->ne[1] > mul_mat_vec_max_cols`. In this fork that constant is the 16-wide array bound, not the working cap, so the check was changed to `ctx->device->mul_mat_vec_max_cols_eff`, to match the fork's per-device GEMV cap. +- `ggml_vk_mul_mat_q_f16`: the fork had moved the buffer lookup block below pipeline selection (graph-plan dynamic subbuffers). The upstream-position copy was removed so it is not declared twice. + +### Vulkan shader type ids - `GGML_TYPE_Q2_0` +Upstream added `vulkan-shaders/ggml_type_ids.glsl` with `#define GGML_TYPE_Q2_0 42u` and moved the FA shaders from `FA_TYPE_*` to `GGML_TYPE_*`. In this fork 42 is `TURBO3_0`, and Q2_0 was renumbered to 56 in the 2026-08 sync (see the NOTE(fork) in `ggml.h`). **Changed the shader define to 56u** and added the fork's `TURBO2/3/4_0` ids there. The fork's `FA_TYPE_Q1_0` / `FA_TYPE_TURBO4_0` cases in `fa_types.glsl` / `flash_attn_dequant.glsl` were renamed to the new `GGML_TYPE_*` names. Without this, a Q2_0 matmul on Vulkan would have used the TURBO3_0 branch. + +## Found by the post-merge build (fixed in the follow-up commit) + +The merge commit was pushed before the compile check finished. Building with clang (CPU + Vulkan + all tests) found these. They are fixed in the commit after the merge. + +- **Router deadlock (important; corrects the `unload_lru()` decision above).** Upstream #28555 removed the router's stopper thread and its `cv_stop`. Stops now go through `request_stop()`, which **returns early when the model is already in `stopping_models`**. The fork's code still used "insert into `stopping_models`, then wake the stopper thread", in `unload_lru()` and in both GPU-placement eviction paths. As merged, a victim would be marked stopping, `unload()` -> `request_stop()` would then no-op, and the router would wait forever for a model that was never told to exit. All three sites now call `request_stop(victim, !loading)` under the same lock that picked the victim. The fork's guarantee (a concurrent `unload_lru()` cannot choose a second victim) still holds, because `request_stop()` marks the victim stopping under that same lock. +- `src/models/models.h`: both sides had picked up upstream #28068 (`build_gdn_l2_norm`) at different spots, so git kept two definitions. Removed one. +- `common/speculative.cpp`: upstream renamed `common_speculative_draft_params::n_past` to `pos0` (#28715, "it is a position, not a count"). Updated the fork's `WP_DSPARK_DEBUG` log lines. +- Vulkan: the fork's `ggml_vk_ensure_host_read_staging_buffer` / `ggml_vk_ensure_wp_fused_batch_scratch_buffer` landed in `ggml-vulkan-buffers.cpp` (next to `ensure_sync_staging_buffer`, where upstream moved it) but are called from `ggml-vulkan.cpp`. Made them shared and declared them in `common.h`. +- Vulkan, **two misplaced replay hunks** (identical context text in two places): + - The fork's `d_X_buf` / `d_Y_buf` dynamic-subbuffer block and the `d_Qy_copy` quantize source belong to `ggml_vk_mul_mat_q_f16` but had landed in `ggml_vk_mul_mat_id_q_f16`. Moved back. + - `case GGML_TYPE_TURBO4_0:` in `supports_op` belongs in the F32->TURBO4_0 (quantize) CPY case but had landed in TURBO4_0->F32. As merged, the backend would have claimed an op it does not implement. Moved back. + To catch any others, every fork-modified function was checked with a per-function 3-way merge (base / fork / upstream) against the ported body. After the fixes, the only differences left are the hand-merged functions described above. + +### Pre-existing on origin/master (not caused by this merge; not changed) +- `ggml/include/ggml-ml8.h` (from `55bc7c6ea`) declares a C++ overload of `ggml_fp8_quant_rot` inside `extern "C"`. Clang accepts it, **GCC rejects it** ("conflicting declaration of C function"). Every build here used clang, which is what the ROCm toolchain uses anyway. + +## Verification + +- Build: clang, `-DGGML_VULKAN=ON -DLLAMA_BUILD_TESTS=ON`, Release. All 1014 targets build (libllama including the unity-built `models/*.cpp`, llama-server, the wp-expert-worker/dispatcher tools, all tests). No CUDA/HIP toolchain was available in the sync environment, so **the `.cu` changes (FA instance selection, mmq tile sizing, BF16 cuBLAS fallback, allreduce) are only checked by reading, not compiled.** Build `-DGGML_HIP=ON` before merging to master. +- `ctest -LE model` on CPU (no GPU in the sync environment): 90/106 pass. All 16 failures were checked against a clean build of pre-merge `origin/master` (`26e5f58`) in a separate worktree: + - 11 fail identically on origin/master, so they are **pre-existing**: test-arg-parser (asserts upstream's `n_outputs_max_per_seq == 1`, which the fork deliberately does not use; see speculative.cpp), test-dsv41-load/-decode, test-recurrent-state-rollback/-dsv4, test-save-load-state, test-paged-decode-oracle, test-wp-expert-worker, test-routed-experts-external, test-quantize-fns, test-ml8-registry. + - 4 need model downloads (no network in the sync environment): test-download-model, test-eval-callback(-download-model), test-thread-safety. + - test-barrier timed out only under `-j4` load. It passes when run alone on both trees. +- Not verified here (needs your hardware): a GPU run of test-backend-ops on ROCm/Vulkan, a router run with `--gpus` placement plus `models_max` eviction (to exercise the deadlock fix), and a DSpark/DFlash draft on a Meta-split target. diff --git a/docs/development/parsing.md b/docs/development/parsing.md index a41057db2b8a..0cb372eca56a 100644 --- a/docs/development/parsing.md +++ b/docs/development/parsing.md @@ -28,7 +28,7 @@ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { for (const auto & tool : tools) { const auto & function = tool.at("function"); std::string name = function.at("name"); - const auto & schema = function.at("parameters"); + const auto schema = common_chat_tool_parameters(function); auto tool_name = p.json_member("name", "\"" + p.literal(name) + "\""); auto tool_args = p.json_member("arguments", p.schema(p.json(), "tool-" + name + "-schema", schema)); @@ -108,6 +108,7 @@ For a more complete example, see `test_example_native()` in - **`rule(name, p, trigger)`** - Creates a named rule and returns a reference - **`trigger_rule(name, p)`** - Creates a trigger rule (entry point for lazy grammar generation) - **`schema(p, name, schema, raw)`** - Wraps parser with JSON schema metadata for grammar generation +- **`schema(p, name, doc, node, raw)`** - Same, for a node of a `common_chat_schema_document` built earlier, e.g. one tool parameter ### AST Control @@ -121,9 +122,6 @@ some exceptions. ```cpp data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(params.tools, [&](const json & fn) { - builder.resolve_refs(fn.at("parameters")); - }); parser.build_grammar(builder, data.grammar_lazy); }); ``` @@ -151,7 +149,8 @@ implementation to generate the grammar instead of the underlying parser. The `raw` option emits a grammar suitable for a raw string instead of a JSON string. In other words, it won't be wrapped in quotes or require escaping -quotes. It should only be used when `type == "string"`. +quotes. It only takes effect when the schema may be a string, as reported by +`common_chat_schema::may_be_string()`, otherwise the JSON grammar is used. The downside is that it can potentially lead to ambiguous grammars. For example, if a user provides the pattern `^.*$`, the following grammar may be diff --git a/docs/ops.md b/docs/ops.md index cc8d253820cc..0921392f3682 100644 --- a/docs/ops.md +++ b/docs/ops.md @@ -44,7 +44,7 @@ Legend: | DSV4_HC_COMB | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | DSV4_HC_POST | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | DSV4_HC_PRE | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| DUP | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ | +| DUP | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ | | ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | @@ -60,7 +60,7 @@ Legend: | GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ | -| GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | +| GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | | GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | | HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | @@ -70,11 +70,11 @@ Legend: | LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | LIGHTNING_INDEXER | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | LOG | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | -| MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | +| MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | ❌ | | MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | | MUL_MAT_HADAMARD | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | -| MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ❌ | +| MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ | | NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ | | OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | @@ -98,7 +98,7 @@ Legend: | RWKV_WKV7 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | -| SET_ROWS | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ | ❌ | +| SET_ROWS | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | | SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | @@ -117,11 +117,11 @@ Legend: | SUM | ❌ | 🟡 | ✅ | 🟡 | ❌ | ❌ | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | ❌ | | SUM_ROWS | ❌ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | | SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | -| SWIGLU_CLAMP | ❌ | ❌ | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| SWIGLU_CLAMP | ❌ | ❌ | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | SWIGLU_OAI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | TIMESTEP_EMBEDDING | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | -| TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | +| TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ | | TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | UPSCALE | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | diff --git a/docs/ops/CUDA.csv b/docs/ops/CUDA.csv index 22c84dd143d9..570bdf75ee56 100644 --- a/docs/ops/CUDA.csv +++ b/docs/ops/CUDA.csv @@ -5000,14 +5000,14 @@ "CUDA0","REPEAT_BACK","type=f32,ne=[8,6,4,2],nr=[1,1,1,2],v=1","support","1","yes","CUDA" "CUDA0","DUP","type=f32,ne=[10,10,20,1]","support","1","yes","CUDA" "CUDA0","DUP","type=f16,ne=[10,10,20,1]","support","1","yes","CUDA" -"CUDA0","DUP","type=i32,ne=[10,10,20,1]","support","0","no","CUDA" -"CUDA0","DUP","type=i16,ne=[10,10,20,1]","support","0","no","CUDA" +"CUDA0","DUP","type=i32,ne=[10,10,20,1]","support","1","yes","CUDA" +"CUDA0","DUP","type=i16,ne=[10,10,20,1]","support","1","yes","CUDA" "CUDA0","DUP","type=f32,ne=[10,10,5,1],permute=[0,2,1,3]","support","1","yes","CUDA" "CUDA0","DUP","type=f16,ne=[10,10,5,1],permute=[0,2,1,3]","support","1","yes","CUDA" "CUDA0","DUP","type=f32,ne=[10,10,5,1],permute=[1,0,2,3]","support","1","yes","CUDA" "CUDA0","DUP","type=f16,ne=[10,10,5,1],permute=[1,0,2,3]","support","1","yes","CUDA" -"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[0,2,1,3]","support","0","no","CUDA" -"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[1,2,0,3]","support","0","no","CUDA" +"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[0,2,1,3]","support","1","yes","CUDA" +"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[1,2,0,3]","support","1","yes","CUDA" "CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=1","support","1","yes","CUDA" "CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=2","support","1","yes","CUDA" "CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=3","support","1","yes","CUDA" diff --git a/docs/ops/Hexagon.csv b/docs/ops/Hexagon.csv index 6709c64b6896..46618a173d58 100644 --- a/docs/ops/Hexagon.csv +++ b/docs/ops/Hexagon.csv @@ -13936,247 +13936,247 @@ "HTP0","ARGSORT","type=f32,ne=[2049,2,1,3],order=1","support","1","yes","HTP" "HTP0","ARGSORT","type=f32,ne=[2,8,8192,1],order=1","support","1","yes","HTP" "HTP0","ARGSORT","type=f32,ne=[2048,512,1,1],order=1","support","1","yes","HTP" -"HTP0","TOP_K","type=f32,ne=[1,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[12,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[13,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[13,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[15,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[15,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[15,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[19,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[19,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[19,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[19,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[27,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[27,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[27,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[27,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[27,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[43,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[43,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[43,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[43,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[43,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[64,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[75,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[64,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[75,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[64,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[75,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[64,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[75,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[64,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[75,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=9999,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=9999,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=9999,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=9999,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=1,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[1,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[12,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[13,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[13,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[15,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[15,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[15,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[19,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[19,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[19,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[19,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[27,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[27,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[27,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[27,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[27,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[43,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[43,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[43,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[43,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[43,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[64,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[75,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[64,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[75,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[64,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[75,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[64,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[75,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[64,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[75,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[128,1,1,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[139,1,2,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[256,1,1,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[267,1,2,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[512,1,1,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[523,1,2,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1035,1,2,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,1,1,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2059,1,2,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4107,1,2,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8203,1,2,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=9999,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16395,1,2,1],k=9999,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=100,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=500,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=1023,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32768,1,1,1],k=9999,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[32779,1,2,1],k=9999,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=1,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[65547,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=2,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=2,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[65547,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=3,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=3,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[65547,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=7,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=7,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[65547,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=15,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=15,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[65547,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=100,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=100,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[65547,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=500,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=500,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[65547,1,2,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=1023,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=1023,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[65547,1,2,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=9999,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[65536,1,1,1],k=9999,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[65547,1,2,1],k=9999,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=1,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=1,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[131083,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=2,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=2,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[131083,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=3,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=3,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[131083,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=7,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=7,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[131083,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=15,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=15,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[131083,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=100,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=100,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[131083,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=500,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=500,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[131083,1,2,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=1023,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=1023,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[131083,1,2,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=9999,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[131072,1,1,1],k=9999,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[131083,1,2,1],k=9999,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=1,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=1,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[262155,1,2,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=2,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=2,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[262155,1,2,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=3,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=3,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[262155,1,2,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=7,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=7,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[262155,1,2,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=15,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=15,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[262155,1,2,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=100,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=100,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[262155,1,2,1],k=100,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=500,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=500,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[262155,1,2,1],k=500,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=1023,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=1023,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[262155,1,2,1],k=1023,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=9999,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[262144,1,1,1],k=9999,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[262155,1,2,1],k=9999,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[524288,1,1,1],k=1,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[524299,1,2,1],k=1,ties=0","support","0","no","HTP" @@ -14196,99 +14196,105 @@ "HTP0","TOP_K","type=f32,ne=[524299,1,2,1],k=1023,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[524288,1,1,1],k=9999,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[524299,1,2,1],k=9999,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[202048,1,1,1],k=4,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[151936,1,1,1],k=4,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=4,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,1,1,1],k=4,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[202048,1,1,1],k=4,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[151936,1,1,1],k=4,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=4,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,1,1,1],k=4,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[202048,8,1,1],k=4,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[151936,8,1,1],k=4,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,8,1,1],k=4,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,8,1,1],k=4,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,8,1,1],k=4,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,8,1,1],k=4,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[202048,16,1,1],k=4,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[151936,16,1,1],k=4,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,16,1,1],k=4,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,16,1,1],k=4,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[202048,1,1,1],k=8,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[151936,1,1,1],k=8,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=8,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,1,1,1],k=8,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,16,1,1],k=4,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,16,1,1],k=4,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[202048,1,1,1],k=8,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[151936,1,1,1],k=8,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=8,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,1,1,1],k=8,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[202048,8,1,1],k=8,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[151936,8,1,1],k=8,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,8,1,1],k=8,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,8,1,1],k=8,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,8,1,1],k=8,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,8,1,1],k=8,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[202048,16,1,1],k=8,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[151936,16,1,1],k=8,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,16,1,1],k=8,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,16,1,1],k=8,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[202048,1,1,1],k=16,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[151936,1,1,1],k=16,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=16,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,1,1,1],k=16,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,16,1,1],k=8,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,16,1,1],k=8,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[202048,1,1,1],k=16,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[151936,1,1,1],k=16,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=16,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,1,1,1],k=16,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[202048,8,1,1],k=16,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[151936,8,1,1],k=16,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,8,1,1],k=16,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,8,1,1],k=16,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,8,1,1],k=16,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,8,1,1],k=16,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[202048,16,1,1],k=16,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[151936,16,1,1],k=16,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,16,1,1],k=16,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,16,1,1],k=16,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[202048,1,1,1],k=32,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[151936,1,1,1],k=32,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=32,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,1,1,1],k=32,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,16,1,1],k=16,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,16,1,1],k=16,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[202048,1,1,1],k=32,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[151936,1,1,1],k=32,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,1,1,1],k=32,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,1,1,1],k=32,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[202048,8,1,1],k=32,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[151936,8,1,1],k=32,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,8,1,1],k=32,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,8,1,1],k=32,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,8,1,1],k=32,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,8,1,1],k=32,ties=0","support","1","yes","HTP" "HTP0","TOP_K","type=f32,ne=[202048,16,1,1],k=32,ties=0","support","0","no","HTP" "HTP0","TOP_K","type=f32,ne=[151936,16,1,1],k=32,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8192,16,1,1],k=32,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[8193,16,1,1],k=32,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16,10,10,10],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[60,10,10,10],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1023,2,1,3],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,2,1,3],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1025,2,1,3],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2047,2,1,3],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,2,1,3],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2049,2,1,3],k=1,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16,10,10,10],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[60,10,10,10],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1023,2,1,3],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,2,1,3],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1025,2,1,3],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2047,2,1,3],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,2,1,3],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2049,2,1,3],k=2,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16,10,10,10],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[60,10,10,10],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1023,2,1,3],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,2,1,3],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1025,2,1,3],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2047,2,1,3],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,2,1,3],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2049,2,1,3],k=3,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16,10,10,10],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[60,10,10,10],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1023,2,1,3],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,2,1,3],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1025,2,1,3],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2047,2,1,3],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,2,1,3],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2049,2,1,3],k=7,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16,10,10,10],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[60,10,10,10],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1023,2,1,3],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1024,2,1,3],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[1025,2,1,3],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2047,2,1,3],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2048,2,1,3],k=15,ties=0","support","0","no","HTP" -"HTP0","TOP_K","type=f32,ne=[2049,2,1,3],k=15,ties=0","support","0","no","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,16,1,1],k=32,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8193,16,1,1],k=32,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16,10,10,10],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[60,10,10,10],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1023,2,1,3],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,2,1,3],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1025,2,1,3],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2047,2,1,3],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,2,1,3],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2049,2,1,3],k=1,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16,10,10,10],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[60,10,10,10],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1023,2,1,3],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,2,1,3],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1025,2,1,3],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2047,2,1,3],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,2,1,3],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2049,2,1,3],k=2,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16,10,10,10],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[60,10,10,10],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1023,2,1,3],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,2,1,3],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1025,2,1,3],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2047,2,1,3],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,2,1,3],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2049,2,1,3],k=3,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16,10,10,10],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[60,10,10,10],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1023,2,1,3],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,2,1,3],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1025,2,1,3],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2047,2,1,3],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,2,1,3],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2049,2,1,3],k=7,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16,10,10,10],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[60,10,10,10],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1023,2,1,3],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,2,1,3],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1025,2,1,3],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[16384,1,1,1],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2047,2,1,3],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,2,1,3],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2049,2,1,3],k=15,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[1024,1,1,1],k=1024,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[2048,2,1,1],k=1024,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[4096,1,1,1],k=2048,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[8192,2,1,1],k=2051,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[33024,1,1,1],k=2051,ties=0","support","1","yes","HTP" +"HTP0","TOP_K","type=f32,ne=[33024,4,1,1],k=2051,ties=0","support","1","yes","HTP" "HTP0","UPSCALE","type=f32,ne=[512,512,3,2],scale_factor=2,mode=nearest,transpose=0","support","0","no","HTP" "HTP0","UPSCALE","type=f32,ne=[512,512,3,2],scale_factor=2,mode=nearest,transpose=1","support","0","no","HTP" "HTP0","UPSCALE","type=f32,ne=[2,5,7,11],ne_tgt=[5,7,11,13],mode=nearest","support","0","no","HTP" diff --git a/docs/ops/SYCL.csv b/docs/ops/SYCL.csv index 5aaaa73456f4..09e698abace2 100644 --- a/docs/ops/SYCL.csv +++ b/docs/ops/SYCL.csv @@ -171,14 +171,37 @@ "SYCL0","DSV4_HC_COMB","n_tokens=17,n_iter=4,eps=0.000001","support","1","yes","SYCL" "SYCL0","DSV4_HC_COMB","n_tokens=257,n_iter=8,eps=0.000001","support","1","yes","SYCL" "SYCL0","DSV4_HC_COMB","n_tokens=17,n_iter=20,eps=0.000001","support","1","yes","SYCL" -"SYCL0","DSV4_HC_PRE","n_embd=1,n_tokens=1","support","1","yes","SYCL" -"SYCL0","DSV4_HC_PRE","n_embd=31,n_tokens=17","support","1","yes","SYCL" -"SYCL0","DSV4_HC_PRE","n_embd=128,n_tokens=257","support","1","yes","SYCL" -"SYCL0","DSV4_HC_PRE","n_embd=4096,n_tokens=21","support","1","yes","SYCL" -"SYCL0","DSV4_HC_POST","n_embd=1,n_tokens=1","support","1","yes","SYCL" -"SYCL0","DSV4_HC_POST","n_embd=31,n_tokens=17","support","1","yes","SYCL" -"SYCL0","DSV4_HC_POST","n_embd=128,n_tokens=257","support","1","yes","SYCL" -"SYCL0","DSV4_HC_POST","n_embd=4096,n_tokens=21","support","1","yes","SYCL" +"SYCL0","DSV4_HC_COMB","n_tokens=1,n_iter=20,eps=0.000001","support","1","yes","SYCL" +"SYCL0","DSV4_HC_COMB","n_tokens=256,n_iter=20,eps=0.000001","support","1","yes","SYCL" +"SYCL0","DSV4_HC_COMB","n_tokens=336,n_iter=20,eps=0.000001","support","1","yes","SYCL" +"SYCL0","DSV4_HC_COMB","n_tokens=512,n_iter=20,eps=0.000001","support","1","yes","SYCL" +"SYCL0","DSV4_HC_COMB","n_tokens=513,n_iter=20,eps=0.000001","support","1","yes","SYCL" +"SYCL0","DSV4_HC_COMB","n_tokens=1024,n_iter=20,eps=0.000001","support","1","yes","SYCL" +"SYCL0","DSV4_HC_COMB","n_tokens=2048,n_iter=20,eps=0.000001","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=1,n_hc=4,n_tokens=1,gated=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=31,n_hc=4,n_tokens=17,gated=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=4,n_tokens=257,gated=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=4096,n_hc=4,n_tokens=21,gated=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=31,n_hc=4,n_tokens=17,gated=1","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=4096,n_hc=4,n_tokens=21,gated=1","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=1,n_tokens=17,gated=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=1,n_tokens=17,gated=1","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=2,n_tokens=17,gated=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=2,n_tokens=17,gated=1","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=3,n_tokens=17,gated=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=3,n_tokens=17,gated=1","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=5,n_tokens=17,gated=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=5,n_tokens=17,gated=1","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=8,n_tokens=17,gated=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=8,n_tokens=17,gated=1","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=65,n_tokens=17,gated=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_PRE","n_embd=128,n_hc=65,n_tokens=17,gated=1","support","1","yes","SYCL" +"SYCL0","DSV4_HC_POST","n_embd=1,n_tokens=1,identity=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_POST","n_embd=31,n_tokens=17,identity=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_POST","n_embd=128,n_tokens=257,identity=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_POST","n_embd=4096,n_tokens=21,identity=0","support","1","yes","SYCL" +"SYCL0","DSV4_HC_POST","n_embd=31,n_tokens=17,identity=1","support","1","yes","SYCL" +"SYCL0","DSV4_HC_POST","n_embd=4096,n_tokens=21,identity=1","support","1","yes","SYCL" "SYCL0","REGLU","type=f16,ne_a=[128,2,2,2],v=0,swapped=0","support","1","yes","SYCL" "SYCL0","REGLU","type=f16,ne_a=[5,7,11,13],v=0,swapped=0","support","1","yes","SYCL" "SYCL0","REGLU","type=f16,ne_a=[128,2,2,2],v=0,swapped=1","support","1","yes","SYCL" @@ -307,131 +330,260 @@ "SYCL0","SWIGLU_OAI","type=f32,ne_a=[128,2,2,2],v=1,alpha=0.500000,limit=7.000000","support","1","yes","SYCL" "SYCL0","SWIGLU_OAI","type=f32,ne_a=[128,2,2,2],v=1,alpha=1.702000,limit=2.000000","support","1","yes","SYCL" "SYCL0","SWIGLU_OAI","type=f32,ne_a=[128,2,2,2],v=1,alpha=1.702000,limit=7.000000","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f32,n=76800,m=5,r=4,be1=1,be2=2,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f32,n=256,m=80000,r=70000,be1=2,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=700,be2=100,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_0,n=76800,m=5,r=4,be1=1,be2=2,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_0,n=256,m=80000,r=70000,be1=2,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=700,be2=100,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f32,n=1,m=8,r=2,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=1,be2=1,v=0","support","0","no","SYCL" -"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=1,be2=1,v=1","support","0","no","SYCL" -"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=7,be2=1,v=0","support","0","no","SYCL" -"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=7,be2=1,v=1","support","0","no","SYCL" -"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=1,be2=1,v=0","support","0","no","SYCL" -"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=1,be2=1,v=1","support","0","no","SYCL" -"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=7,be2=1,v=0","support","0","no","SYCL" -"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=7,be2=1,v=1","support","0","no","SYCL" -"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","SYCL" -"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","SYCL" -"SYCL0","GET_ROWS_BACK","type=f32,n=1,m=8,r=2,b=1,v=0","support","0","no","SYCL" -"SYCL0","GET_ROWS_BACK","type=f32,n=1,m=70000,r=4,b=1,v=0","support","0","no","SYCL" -"SYCL0","GET_ROWS_BACK","type=f32,n=256,m=5,r=4,b=1,v=0","support","0","no","SYCL" -"SYCL0","GET_ROWS_BACK","type=f32,n=256,m=5,r=4,b=1,v=1","support","0","no","SYCL" -"SYCL0","GET_ROWS_BACK","type=f16,n=256,m=5,r=4,b=1,v=0","support","0","no","SYCL" -"SYCL0","GET_ROWS_BACK","type=f16,n=256,m=5,r=4,b=1,v=1","support","0","no","SYCL" +"SYCL0","SWIGLU_CLAMP","type=f16,ne_a=[128,2,2,2],v=0,limit=2.000000","support","1","yes","SYCL" +"SYCL0","SWIGLU_CLAMP","type=f16,ne_a=[128,2,2,2],v=0,limit=10.000000","support","1","yes","SYCL" +"SYCL0","SWIGLU_CLAMP","type=f16,ne_a=[128,2,2,2],v=1,limit=2.000000","support","1","yes","SYCL" +"SYCL0","SWIGLU_CLAMP","type=f16,ne_a=[128,2,2,2],v=1,limit=10.000000","support","1","yes","SYCL" +"SYCL0","SWIGLU_CLAMP","type=f32,ne_a=[128,2,2,2],v=0,limit=2.000000","support","1","yes","SYCL" +"SYCL0","SWIGLU_CLAMP","type=f32,ne_a=[128,2,2,2],v=0,limit=10.000000","support","1","yes","SYCL" +"SYCL0","SWIGLU_CLAMP","type=f32,ne_a=[128,2,2,2],v=1,limit=2.000000","support","1","yes","SYCL" +"SYCL0","SWIGLU_CLAMP","type=f32,ne_a=[128,2,2,2],v=1,limit=10.000000","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=76800,m=5,r=4,be1=1,be2=2,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=256,m=80000,r=70000,be1=2,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=700,be2=100,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_0,n=76800,m=5,r=4,be1=1,be2=2,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_0,n=256,m=80000,r=70000,be1=2,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=700,be2=100,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=1,m=8,r=2,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f16,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=bf16,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_1,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_1,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q8_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=nvfp4,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q2_K,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q3_K,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q4_K,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q5_K,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=q6_K,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq2_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq1_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq1_0,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq1_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq1_0,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq1_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq1_0,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq1_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=tq1_0,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","0","no","SYCL" +"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xxs,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_xs,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq2_s,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_xxs,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_s,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq1_m,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_nl,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq3_s,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=iq4_xs,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=1,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=1,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=7,be2=1,v=0,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=0,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=7,be2=1,v=1,vs0=1,offset_cols=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS","type=f32,n=256,m=8,r=2,be1=1,be2=1,v=0,vs0=1,offset_cols=3","support","1","yes","SYCL" +"SYCL0","GET_ROWS_BACK","type=f32,n=1,m=8,r=2,b=1,v=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS_BACK","type=f32,n=1,m=70000,r=4,b=1,v=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS_BACK","type=f32,n=256,m=5,r=4,b=1,v=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS_BACK","type=f32,n=256,m=5,r=4,b=1,v=1","support","1","yes","SYCL" +"SYCL0","GET_ROWS_BACK","type=f16,n=256,m=5,r=4,b=1,v=0","support","1","yes","SYCL" +"SYCL0","GET_ROWS_BACK","type=f16,n=256,m=5,r=4,b=1,v=1","support","1","yes","SYCL" "SYCL0","GET_ROWS_BACK","type=bf16,n=256,m=5,r=4,b=1,v=0","support","0","no","SYCL" "SYCL0","GET_ROWS_BACK","type=bf16,n=256,m=5,r=4,b=1,v=1","support","0","no","SYCL" "SYCL0","GET_ROWS_BACK","type=q4_0,n=256,m=5,r=4,b=1,v=0","support","0","no","SYCL" @@ -464,6 +616,8 @@ "SYCL0","GET_ROWS_BACK","type=q6_K,n=256,m=5,r=4,b=1,v=1","support","0","no","SYCL" "SYCL0","GET_ROWS_BACK","type=tq2_0,n=256,m=5,r=4,b=1,v=0","support","0","no","SYCL" "SYCL0","GET_ROWS_BACK","type=tq2_0,n=256,m=5,r=4,b=1,v=1","support","0","no","SYCL" +"SYCL0","GET_ROWS_BACK","type=tq1_0,n=256,m=5,r=4,b=1,v=0","support","0","no","SYCL" +"SYCL0","GET_ROWS_BACK","type=tq1_0,n=256,m=5,r=4,b=1,v=1","support","0","no","SYCL" "SYCL0","GET_ROWS_BACK","type=iq2_xxs,n=256,m=5,r=4,b=1,v=0","support","0","no","SYCL" "SYCL0","GET_ROWS_BACK","type=iq2_xxs,n=256,m=5,r=4,b=1,v=1","support","0","no","SYCL" "SYCL0","GET_ROWS_BACK","type=iq2_xs,n=256,m=5,r=4,b=1,v=0","support","0","no","SYCL" @@ -715,18 +869,30 @@ "SYCL0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","SYCL" "SYCL0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","SYCL" "SYCL0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq2_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f16,type_dst=tq1_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","SYCL" "SYCL0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","SYCL" "SYCL0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","SYCL" "SYCL0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","SYCL" @@ -1063,18 +1229,30 @@ "SYCL0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","SYCL" "SYCL0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","SYCL" "SYCL0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","SYCL" -"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq2_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","SYCL" +"SYCL0","SET_ROWS","type_src=f32,type_dst=tq1_0,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","SYCL" "SYCL0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","SYCL" "SYCL0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","SYCL" "SYCL0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","SYCL" @@ -6814,6 +6992,9 @@ "SYCL0","CONV_2D","ne_input=[256,256,192,1],ne_kernel=[3,3,192,96],type_kernel=f32,stride0=1,stride1=1,padding0=1,padding1=1,dilation0=1,dilation1=1,cwhn=1","support","1","yes","SYCL" "SYCL0","CONV_2D","ne_input=[256,256,192,1],ne_kernel=[3,3,192,96],type_kernel=f16,stride0=1,stride1=1,padding0=1,padding1=1,dilation0=1,dilation1=1,cwhn=0","support","1","yes","SYCL" "SYCL0","CONV_2D","ne_input=[256,256,192,1],ne_kernel=[3,3,192,96],type_kernel=f16,stride0=1,stride1=1,padding0=1,padding1=1,dilation0=1,dilation1=1,cwhn=1","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[19,17,8,2],ne_kernel=[3,3,8,65],type_kernel=f16,stride0=1,stride1=1,padding0=1,padding1=1,dilation0=1,dilation1=1,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[19,17,16,3],ne_kernel=[3,3,16,33],type_kernel=f16,stride0=2,stride1=3,padding0=4,padding1=2,dilation0=2,dilation1=1,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[13,11,16,3],ne_kernel=[1,1,16,33],type_kernel=f16,stride0=1,stride1=1,padding0=0,padding1=0,dilation0=1,dilation1=1,cwhn=0","support","1","yes","SYCL" "SYCL0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],type_kernel=f32,stride=1,padding=0,dilation=1,cwhn=0","support","1","yes","SYCL" "SYCL0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],type_kernel=f32,stride=1,padding=0,dilation=1,cwhn=1","support","1","yes","SYCL" "SYCL0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],type_kernel=f32,stride=2,padding=1,dilation=1,cwhn=0","support","1","yes","SYCL" @@ -7232,9 +7413,11 @@ "SYCL0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[3,2,3,1],ne_kernel=[2,2,1,3],stride=1","support","1","yes","SYCL" "SYCL0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[10,10,9,1],ne_kernel=[3,3,1,9],stride=2","support","1","yes","SYCL" "SYCL0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[129,63,35,1],ne_kernel=[3,3,48,35],stride=1","support","1","yes","SYCL" +"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[10,10,9,2],ne_kernel=[3,3,1,9],stride=2","support","1","yes","SYCL" "SYCL0","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[3,2,3,1],ne_kernel=[2,2,1,3],stride=1","support","1","yes","SYCL" "SYCL0","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[10,10,9,1],ne_kernel=[3,3,1,9],stride=2","support","1","yes","SYCL" "SYCL0","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[129,63,35,1],ne_kernel=[3,3,48,35],stride=1","support","1","yes","SYCL" +"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[10,10,9,2],ne_kernel=[3,3,1,9],stride=2","support","1","yes","SYCL" "SYCL0","COUNT_EQUAL","type=f32,ne=[4,500,1,1]","support","1","yes","SYCL" "SYCL0","COUNT_EQUAL","type=f32,ne=[4,5000,1,1]","support","1","yes","SYCL" "SYCL0","ARGMAX","type=f32,ne=[32,1,1,1]","support","1","yes","SYCL" @@ -7447,15 +7630,24 @@ "SYCL0","CPY","type_src=q6_K,type_dst=q6_K,ne_src=[768,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" "SYCL0","CPY","type_src=q6_K,type_dst=q6_K,ne_src=[768,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" "SYCL0","CPY","type_src=q6_K,type_dst=q6_K,ne_src=[768,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[512,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[512,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[512,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[768,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[768,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[768,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","1","yes","SYCL" +"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[512,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[512,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[512,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[768,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[768,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq2_0,type_dst=tq2_0,ne_src=[768,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq1_0,type_dst=tq1_0,ne_src=[256,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq1_0,type_dst=tq1_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq1_0,type_dst=tq1_0,ne_src=[256,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq1_0,type_dst=tq1_0,ne_src=[512,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq1_0,type_dst=tq1_0,ne_src=[512,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq1_0,type_dst=tq1_0,ne_src=[512,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq1_0,type_dst=tq1_0,ne_src=[768,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq1_0,type_dst=tq1_0,ne_src=[768,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq1_0,type_dst=tq1_0,ne_src=[768,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=iq2_xxs,type_dst=iq2_xxs,ne_src=[256,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=iq2_xxs,type_dst=iq2_xxs,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=iq2_xxs,type_dst=iq2_xxs,ne_src=[256,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","SYCL" @@ -7571,8 +7763,10 @@ "SYCL0","CPY","type_src=f16,type_dst=q5_K,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=f16,type_dst=q6_K,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=f16,type_dst=q6_K,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" -"SYCL0","CPY","type_src=f16,type_dst=tq2_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=f16,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" +"SYCL0","CPY","type_src=f16,type_dst=tq2_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=f16,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=f16,type_dst=tq1_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=f16,type_dst=tq1_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=f16,type_dst=iq2_xxs,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=f16,type_dst=iq2_xxs,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=f16,type_dst=iq2_xs,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" @@ -7625,8 +7819,10 @@ "SYCL0","CPY","type_src=bf16,type_dst=q5_K,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=bf16,type_dst=q6_K,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=bf16,type_dst=q6_K,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" -"SYCL0","CPY","type_src=bf16,type_dst=tq2_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=bf16,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" +"SYCL0","CPY","type_src=bf16,type_dst=tq2_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=bf16,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=bf16,type_dst=tq1_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=bf16,type_dst=tq1_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=bf16,type_dst=iq2_xxs,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=bf16,type_dst=iq2_xxs,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=bf16,type_dst=iq2_xs,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" @@ -7679,8 +7875,10 @@ "SYCL0","CPY","type_src=f32,type_dst=q5_K,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=f32,type_dst=q6_K,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=f32,type_dst=q6_K,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" -"SYCL0","CPY","type_src=f32,type_dst=tq2_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=f32,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" +"SYCL0","CPY","type_src=f32,type_dst=tq2_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=f32,type_dst=tq2_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=f32,type_dst=tq1_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=f32,type_dst=tq1_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=f32,type_dst=iq2_xxs,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=f32,type_dst=iq2_xxs,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=f32,type_dst=iq2_xs,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" @@ -7733,8 +7931,10 @@ "SYCL0","CPY","type_src=q5_K,type_dst=f32,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=q6_K,type_dst=f32,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=q6_K,type_dst=f32,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" -"SYCL0","CPY","type_src=tq2_0,type_dst=f32,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CPY","type_src=tq2_0,type_dst=f32,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" +"SYCL0","CPY","type_src=tq2_0,type_dst=f32,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq2_0,type_dst=f32,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq1_0,type_dst=f32,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" +"SYCL0","CPY","type_src=tq1_0,type_dst=f32,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=iq2_xxs,type_dst=f32,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=iq2_xxs,type_dst=f32,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" "SYCL0","CPY","type_src=iq2_xs,type_dst=f32,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","SYCL" @@ -7875,42 +8075,84 @@ "SYCL0","CPY","type_src=f16,type_dst=f16,ne_src=[32,7,3,5],ne_dst=[3,5,7,32],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" "SYCL0","CPY","type_src=f16,type_dst=f16,ne_src=[32,7,3,5],ne_dst=[32,7,5,3],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" "SYCL0","CPY","type_src=f16,type_dst=f16,ne_src=[32,7,5,3],ne_dst=[3,5,7,32],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[2,1,1,1],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[2,1,3,5],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[2,3,5,7],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[1,4,4,1],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[1,8,17,1],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[10,10,10,1],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[2,1,1,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[2,1,3,5],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[2,3,5,7],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[1,4,4,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[1,8,17,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f32,ne=[10,10,10,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[2,1,1,1],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[2,1,3,5],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[2,3,5,7],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[1,4,4,1],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[1,8,17,1],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[10,10,10,1],use_view_slice=1","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[2,1,1,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[2,1,3,5],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[2,3,5,7],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[1,4,4,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[1,8,17,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=i32,ne=[10,10,10,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f16,ne=[2,1,1,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f16,ne=[2,1,3,5],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f16,ne=[2,3,5,7],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f16,ne=[1,4,4,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f16,ne=[1,8,17,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=f16,ne=[10,10,10,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=bf16,ne=[2,1,1,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=bf16,ne=[2,1,3,5],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=bf16,ne=[2,3,5,7],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=bf16,ne=[1,4,4,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=bf16,ne=[1,8,17,1],use_view_slice=0","support","1","yes","SYCL" -"SYCL0","CONT","type=bf16,ne=[10,10,10,1],use_view_slice=0","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2,1,1,1],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2,1,3,5],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2,3,5,7],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[1,4,4,1],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[1,8,17,1],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[10,10,10,1],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2,1,1,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2,1,3,5],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2,3,5,7],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[1,4,4,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[1,8,17,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[10,10,10,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[2,1,1,1],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[2,1,3,5],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[2,3,5,7],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[1,4,4,1],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[1,8,17,1],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[10,10,10,1],use_view_slice=1,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[2,1,1,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[2,1,3,5],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[2,3,5,7],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[1,4,4,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[1,8,17,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=i32,ne=[10,10,10,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[2,1,1,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[2,1,3,5],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[2,3,5,7],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[1,4,4,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[1,8,17,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[10,10,10,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=bf16,ne=[2,1,1,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=bf16,ne=[2,1,3,5],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=bf16,ne=[2,3,5,7],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=bf16,ne=[1,4,4,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=bf16,ne=[1,8,17,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=bf16,ne=[10,10,10,1],use_view_slice=0,permute=[0,0,0,0]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[10,10,10,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[10,10,10,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[10,10,10,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[33,5,7,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[33,5,7,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[33,5,7,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[64,3,65,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[64,3,65,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[64,3,65,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2,3,5,7],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2,3,5,7],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2,3,5,7],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[1024,64,64,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[1024,64,64,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[1024,64,64,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2304,64,64,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2304,64,64,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[2304,64,64,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[1000,33,65,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[1000,33,65,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f32,ne=[1000,33,65,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[10,10,10,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[10,10,10,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[10,10,10,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[33,5,7,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[33,5,7,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[33,5,7,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[64,3,65,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[64,3,65,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[64,3,65,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[2,3,5,7],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[2,3,5,7],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[2,3,5,7],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[1024,64,64,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[1024,64,64,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[1024,64,64,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[2304,64,64,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[2304,64,64,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[2304,64,64,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[1000,33,65,1],use_view_slice=0,permute=[2,1,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[1000,33,65,1],use_view_slice=0,permute=[1,2,0,3]","support","1","yes","SYCL" +"SYCL0","CONT","type=f16,ne=[1000,33,65,1],use_view_slice=0,permute=[0,2,1,3]","support","1","yes","SYCL" "SYCL0","ADD","type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1,perm1=0,src_overlap=0","support","1","yes","SYCL" "SYCL0","SUB","type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1,perm1=0,src_overlap=0","support","1","yes","SYCL" "SYCL0","MUL","type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1,perm1=0,src_overlap=0","support","1","yes","SYCL" @@ -8472,13 +8714,19 @@ "SYCL0","SSM_CONV","type=f32,ne_a=[9,2048,4,1],ne_b=[9,2048,1,1]","support","1","yes","SYCL" "SYCL0","SSM_CONV","type=f32,ne_a=[72,2048,1,1],ne_b=[9,2048,1,1]","support","1","yes","SYCL" "SYCL0","SSM_CONV","type=f32,ne_a=[72,2048,4,1],ne_b=[9,2048,1,1]","support","1","yes","SYCL" -"SYCL0","SSM_SCAN","type=f32,d_state=16,head_dim=1,n_head=1024,n_group=1,n_seq_tokens=32,n_seqs=4,xbc_overlap=0","support","0","no","SYCL" -"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=16,n_group=2,n_seq_tokens=32,n_seqs=4,xbc_overlap=0","support","1","yes","SYCL" -"SYCL0","SSM_SCAN","type=f32,d_state=256,head_dim=64,n_head=8,n_group=2,n_seq_tokens=32,n_seqs=4,xbc_overlap=0","support","1","yes","SYCL" -"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=128,n_head=4,n_group=4,n_seq_tokens=16,n_seqs=2,xbc_overlap=1","support","1","yes","SYCL" -"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=80,n_head=128,n_group=1,n_seq_tokens=256,n_seqs=1,xbc_overlap=0","support","1","yes","SYCL" -"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=80,n_head=128,n_group=1,n_seq_tokens=512,n_seqs=1,xbc_overlap=0","support","1","yes","SYCL" -"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=80,n_group=8,n_seq_tokens=300,n_seqs=2,xbc_overlap=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=16,head_dim=1,n_head=1024,n_group=1,n_seq_tokens=32,n_seqs=4,xbc_overlap=0,K=1,weak_decay=0","support","0","no","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=16,n_group=2,n_seq_tokens=32,n_seqs=4,xbc_overlap=0,K=1,weak_decay=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=256,head_dim=64,n_head=8,n_group=2,n_seq_tokens=32,n_seqs=4,xbc_overlap=0,K=1,weak_decay=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=128,n_head=4,n_group=4,n_seq_tokens=16,n_seqs=2,xbc_overlap=1,K=1,weak_decay=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=80,n_head=128,n_group=1,n_seq_tokens=256,n_seqs=1,xbc_overlap=0,K=1,weak_decay=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=80,n_head=128,n_group=1,n_seq_tokens=512,n_seqs=1,xbc_overlap=0,K=1,weak_decay=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=80,n_group=8,n_seq_tokens=300,n_seqs=2,xbc_overlap=0,K=1,weak_decay=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=16,n_group=2,n_seq_tokens=4,n_seqs=2,xbc_overlap=0,K=4,weak_decay=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=16,n_group=2,n_seq_tokens=8,n_seqs=2,xbc_overlap=0,K=3,weak_decay=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=16,n_group=2,n_seq_tokens=64,n_seqs=4,xbc_overlap=0,K=1,weak_decay=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=16,n_group=2,n_seq_tokens=65,n_seqs=2,xbc_overlap=0,K=1,weak_decay=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=16,n_group=2,n_seq_tokens=128,n_seqs=2,xbc_overlap=0,K=1,weak_decay=0","support","1","yes","SYCL" +"SYCL0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=16,n_group=2,n_seq_tokens=128,n_seqs=2,xbc_overlap=0,K=1,weak_decay=1","support","1","yes","SYCL" "SYCL0","RWKV_WKV6","type=f32,head_count=32,head_size=64,n_seq_tokens=1,n_seqs=1","support","1","yes","SYCL" "SYCL0","RWKV_WKV6","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=1","support","1","yes","SYCL" "SYCL0","RWKV_WKV6","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=4","support","1","yes","SYCL" @@ -8492,2442 +8740,2640 @@ "SYCL0","GATED_LINEAR_ATTN","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=1","support","1","yes","SYCL" "SYCL0","GATED_LINEAR_ATTN","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=4","support","1","yes","SYCL" "SYCL0","GATED_LINEAR_ATTN","type=f32,head_count=32,head_size=64,n_seq_tokens=128,n_seqs=4","support","1","yes","SYCL" -"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=4,k=128,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=32,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=1024,n=1,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=2880,n=32,k=2880,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=2880,n=32,k=2880,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=2880,n=32,k=2880,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=1,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=7,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=8,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=9,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=16,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=128,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=512,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=1,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=64,n=2,k=128,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=83,n=2,k=128,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=64,n=2,k=64,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=83,n=2,k=64,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=64,n=45,k=128,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=45,k=64,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=193,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=67,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=64,n=77,k=77,bs=[12,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=576,n=512,k=576,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=1,n=2048,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=6,n=4096,k=5120,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","SYCL" -"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=16,n_used=16,b=0,m=32,n=1024,k=16","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=2,n_used=2,b=0,m=32,n=8192,k=64","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=16,n_used=16,b=0,m=50,n=200,k=64","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=16,n_used=16,b=0,m=32,n=1024,k=16","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=16,n_used=16,b=0,m=50,n=200,k=64","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=16,n_used=16,b=1,m=32,n=1024,k=16","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=2,n_used=2,b=1,m=32,n=8192,k=64","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=16,n_used=16,b=1,m=50,n=200,k=64","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=16,n_used=16,b=1,m=32,n=1024,k=16","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=16,n_used=16,b=1,m=50,n=200,k=64","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=1,n_used=1,b=0,m=8,n=16,k=1","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=32,n_used=2,b=0,m=2880,n=32,k=2880","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=32,n_used=2,b=0,m=2880,n=32,k=2880","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=3","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=3","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=3","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q5_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q5_1,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=384","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=192","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=192","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q3_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q5_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q6_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=tq2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_s,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq3_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq1_s,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq1_m,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq4_nl,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq4_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q5_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q5_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q5_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q5_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q2_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q3_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q3_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q5_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q5_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q6_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=q6_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=tq2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=tq2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq2_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq3_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq3_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq1_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq1_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq1_m,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq1_m,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq4_nl,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq4_nl,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq4_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=iq4_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","SYCL" -"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=4,k=128,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=32,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=1024,n=1,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=4,k=128,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=4096,n=1,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=1023,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=4096,n=2,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=1023,n=2,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=4096,n=3,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=1023,n=3,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=4096,n=4,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=1023,n=4,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=4096,n=5,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=1023,n=5,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=4096,n=6,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=1023,n=6,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=4096,n=7,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=1023,n=7,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=4096,n=8,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=1023,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=4096,n=1,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=1023,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=4096,n=2,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=1023,n=2,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=4096,n=3,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=1023,n=3,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=4096,n=4,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=1023,n=4,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=4096,n=5,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=1023,n=5,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=4096,n=6,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=1023,n=6,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=4096,n=7,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=1023,n=7,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=4096,n=8,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=1023,n=8,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=6271,n=2,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=6272,n=2,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=6273,n=2,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=2880,n=32,k=2880,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=2880,n=32,k=2880,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=2880,n=32,k=2880,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=1,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=7,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=8,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=9,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=16,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=127,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=128,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=511,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=512,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1,n=512,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=1,n=512,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1,n=509,k=2051,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=1,n=509,k=2051,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=31,n=509,k=2051,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=32,n=509,k=2112,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=32,n=509,k=2112,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=15,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=16,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=31,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=33,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=47,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=48,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=49,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=15,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=16,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=31,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=33,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=47,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=48,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=49,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=15,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=16,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=31,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=33,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=47,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=48,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=49,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=1,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=10,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=16,n=10,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=16,n=10,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=16,n=10,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=16,n=10,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=16,n=10,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=16,n=10,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=10,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=64,n=2,k=128,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=83,n=2,k=128,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=64,n=2,k=64,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=83,n=2,k=64,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=64,n=45,k=128,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=45,k=64,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=193,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=67,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=64,n=77,k=77,bs=[12,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=32,n=4,k=96,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=1","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=576,n=512,k=576,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=1,n=2048,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_1,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_1,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q1_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=mxfp4,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=nvfp4,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q2_K,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q3_K,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q4_K,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q5_K,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q6_K,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=tq2_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=tq1_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_xs,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq2_s,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_xxs,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_s,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq1_m,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_nl,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq3_s,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=q8_0,type_b=f32,m=6,n=4096,k=5120,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=64,n=32,k=65,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=64,n=32,k=80,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=64,n=32,k=80,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=64,n=32,k=80,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=64,n=32,k=588,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f16,m=64,n=32,k=80,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[1,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[1,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[2,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[2,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[4,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[4,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[1,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[1,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[2,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[2,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[4,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[4,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[1,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1056,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=128,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=128,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1056,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2112,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","0","no","SYCL" +"SYCL0","MUL_MAT","type_a=f16,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=bf16,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=1057,n=1,k=129,bs=[8,3],nr=[4,1],per=[0,2,1,3],k_v=0,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT","type_a=f32,type_b=f32,m=129,n=1,k=1057,bs=[8,3],nr=[4,1],per=[0,1,2,3],k_v=2113,o=1,src_overlap=0","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=16,n_used=16,b=0,m=32,n=1024,k=16,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=2,n_used=2,b=0,m=32,n=8192,k=64,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=16,n_used=16,b=0,m=50,n=200,k=64,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=16,n_used=16,b=0,m=32,n=1024,k=16,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=16,n_used=16,b=0,m=50,n=200,k=64,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=16,n_used=16,b=1,m=32,n=1024,k=16,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=2,n_used=2,b=1,m=32,n=8192,k=64,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=16,n_used=16,b=1,m=50,n=200,k=64,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=16,n_used=16,b=1,m=32,n=1024,k=16,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=16,n_used=16,b=1,m=50,n=200,k=64,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=1,n_used=1,b=0,m=1,n=8192,k=4096,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=1,n_used=1,b=0,m=8,n=16,k=1,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=1,n_used=1,b=0,m=8,n=16,k=63,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=1,n_used=1,b=0,m=8,n=16,k=65,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=32,n_used=2,b=0,m=2880,n=32,k=2880,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=512,n_used=10,b=0,m=128,n=1,k=512,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=512,n_used=10,b=0,m=256,n=1,k=128,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=1024,n_used=10,b=0,m=128,n=1,k=512,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=1024,n_used=10,b=0,m=256,n=1,k=128,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=512,n_used=10,b=0,m=128,n=5,k=512,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=512,n_used=10,b=0,m=256,n=5,k=128,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=1024,n_used=10,b=0,m=128,n=5,k=512,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=1024,n_used=10,b=0,m=256,n=5,k=128,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=512,n_used=10,b=0,m=128,n=64,k=512,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=512,n_used=10,b=0,m=256,n=64,k=128,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=1024,n_used=10,b=0,m=128,n=64,k=512,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=1024,n_used=10,b=0,m=256,n=64,k=128,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=512,n_used=10,b=0,m=128,n=300,k=512,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=512,n_used=10,b=0,m=256,n=300,k=128,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=1024,n_used=10,b=0,m=128,n=300,k=512,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=1024,n_used=10,b=0,m=256,n=300,k=128,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=32,n_used=2,b=0,m=2880,n=32,k=2880,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=tq1_0,type_b=f32,n_mats=28,n_used=10,b=0,m=1024,n=1,k=4096,amax=1.000000","support","0","no","SYCL" +"SYCL0","MUL_MAT_ID","type_a=tq1_0,type_b=f32,n_mats=128,n_used=8,b=0,m=1024,n=1,k=2048,amax=1.000000","support","0","no","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=3,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=3,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=3,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q5_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q5_1,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=384,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=192,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=192,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q3_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q5_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q6_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=tq2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","0","no","SYCL" +"SYCL0","MUL_MAT_ID","type_a=tq1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","0","no","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_s,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq1_s,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq1_m,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq4_nl,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq4_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=16,n=10,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=16,n=10,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_s,type_b=f32,n_mats=4,n_used=4,b=0,m=16,n=10,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=16,n=10,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=4,n_used=4,b=0,m=16,n=10,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq1_s,type_b=f32,n_mats=4,n_used=4,b=0,m=16,n=10,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq1_m,type_b=f32,n_mats=4,n_used=4,b=0,m=16,n=10,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq4_xs,type_b=f32,n_mats=4,n_used=4,b=0,m=16,n=10,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q5_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q5_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q5_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q5_1,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q2_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q3_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q3_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q5_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q5_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q6_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=q6_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=tq2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","0","no","SYCL" +"SYCL0","MUL_MAT_ID","type_a=tq2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","0","no","SYCL" +"SYCL0","MUL_MAT_ID","type_a=tq1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","0","no","SYCL" +"SYCL0","MUL_MAT_ID","type_a=tq1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","0","no","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq2_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq1_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq1_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq1_m,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq1_m,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq4_nl,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq4_nl,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq4_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=iq4_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256,amax=1.000000","support","1","yes","SYCL" +"SYCL0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256,amax=1.000000","support","1","yes","SYCL" "SYCL0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[1,1],trans_b=0","support","1","yes","SYCL" "SYCL0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[1,2],trans_b=0","support","1","yes","SYCL" "SYCL0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[2,1],trans_b=0","support","1","yes","SYCL" @@ -12659,6 +13105,8 @@ "SYCL0","SOFT_MAX","type=f32,ne=[200001,2,3,1],mask=1,sinks=1,m_prec=f16,nr23=[1,1],scale=0.100000,max_bias=8.000000,inplace=0","support","1","yes","SYCL" "SYCL0","SOFT_MAX","type=f32,ne=[200000,1,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","SYCL" "SYCL0","SOFT_MAX","type=f32,ne=[200000,4,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","SYCL" +"SYCL0","SOFT_MAX","type=f32,ne=[4,1,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","SYCL" +"SYCL0","SOFT_MAX","type=f32,ne=[4,1023,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","SYCL" "SYCL0","SOFT_MAX","type=f32,ne=[643251,3,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","SYCL" "SYCL0","SOFT_MAX_BACK","type=f32,ne=[16,16,1,1],scale=1.000000,max_bias=0.000000","support","1","yes","SYCL" "SYCL0","SOFT_MAX_BACK","type=f32,ne=[15,15,1,1],scale=1.000000,max_bias=0.000000","support","1","yes","SYCL" @@ -12708,842 +13156,880 @@ "SYCL0","SOFT_MAX_BACK","type=f32,ne=[1024,1024,1,1],scale=0.100000,max_bias=8.000000","support","0","no","SYCL" "SYCL0","SOFT_MAX_BACK","type=f32,ne=[1023,1023,1,1],scale=0.100000,max_bias=8.000000","support","0","no","SYCL" "SYCL0","SOFT_MAX_BACK","type=f32,ne=[1024,1024,2,3],scale=0.100000,max_bias=8.000000","support","0","no","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" -"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,40,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,52,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,64,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,1,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,71,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,8,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=20,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,32,4,1],n_dims=32,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=20,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,28,2,1],n_dims=32,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[80,16,2,1],n_dims=80,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,16,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[16,16,8192,1],n_dims=16,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.000000,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.000000,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.000000,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[64,128,2,1],n_dims=64,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[36,16,2457,1],n_dims=36,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=2,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,3],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=24,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=24,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=24,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,12,2,1],n_dims=24,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=24,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=24,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=24,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f32,ne_a=[128,12,2,1],n_dims=24,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[128,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,12,2,1],n_dims=24,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,12,2,1],n_dims=24,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,12,2,1],n_dims=24,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,12,2,1],n_dims=24,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,12,2,1],n_dims=24,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,12,2,1],n_dims=24,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=32,mode=0,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,12,2,1],n_dims=24,mode=8,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE_BACK","type=f16,ne_a=[128,12,2,1],n_dims=24,mode=40,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=0,inplace=0,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=32,mode=2,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=0,v=0,inplace=1,n_offs=32","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[256,8,512,1],n_dims=64,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[256,16,512,1],n_dims=64,mode=40,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[256,8,512,1],n_dims=256,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=0,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" +"SYCL0","ROPE","type=f32,ne_a=[512,8,512,1],n_dims=128,mode=2,n_ctx=512,fs=1.000000,ef=0.000000,af=1.000000,ff=1,v=0,inplace=0,n_offs=0","support","1","yes","SYCL" "SYCL0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=0","support","1","yes","SYCL" "SYCL0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=0","support","1","yes","SYCL" "SYCL0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=0","support","1","yes","SYCL" @@ -13894,8 +14380,8 @@ "SYCL0","TOP_K","type=f32,ne=[139,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[128,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[139,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[128,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[139,1,2,1],k=100,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[128,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[139,1,2,1],k=100,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[256,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[267,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[256,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -13906,8 +14392,8 @@ "SYCL0","TOP_K","type=f32,ne=[267,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[256,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[267,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[256,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[267,1,2,1],k=100,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[256,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[267,1,2,1],k=100,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[512,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[523,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[512,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -13918,10 +14404,10 @@ "SYCL0","TOP_K","type=f32,ne=[523,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[512,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[523,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[512,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[523,1,2,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[512,1,1,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[523,1,2,1],k=500,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[512,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[523,1,2,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[512,1,1,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[523,1,2,1],k=500,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[1024,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[1035,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[1024,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -13932,12 +14418,12 @@ "SYCL0","TOP_K","type=f32,ne=[1035,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[1024,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[1035,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[1024,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[1035,1,2,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[1024,1,1,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[1035,1,2,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[1024,1,1,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[1035,1,2,1],k=1023,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[1024,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[1035,1,2,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[1024,1,1,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[1035,1,2,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[1024,1,1,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[1035,1,2,1],k=1023,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[2048,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[2059,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[2048,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -13948,12 +14434,12 @@ "SYCL0","TOP_K","type=f32,ne=[2059,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[2048,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[2059,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[2048,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[2059,1,2,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[2048,1,1,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[2059,1,2,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[2048,1,1,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[2059,1,2,1],k=1023,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[2048,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[2059,1,2,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[2048,1,1,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[2059,1,2,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[2048,1,1,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[2059,1,2,1],k=1023,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[4096,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[4107,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[4096,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -13964,12 +14450,12 @@ "SYCL0","TOP_K","type=f32,ne=[4107,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[4096,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[4107,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[4096,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[4107,1,2,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[4096,1,1,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[4107,1,2,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[4096,1,1,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[4107,1,2,1],k=1023,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[4096,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[4107,1,2,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[4096,1,1,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[4107,1,2,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[4096,1,1,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[4107,1,2,1],k=1023,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[8203,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -13980,12 +14466,12 @@ "SYCL0","TOP_K","type=f32,ne=[8203,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[8203,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[8203,1,2,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[8203,1,2,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[8203,1,2,1],k=1023,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8203,1,2,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8203,1,2,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8203,1,2,1],k=1023,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[16384,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[16395,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[16384,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -13996,14 +14482,14 @@ "SYCL0","TOP_K","type=f32,ne=[16395,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[16384,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[16395,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[16384,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[16395,1,2,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[16384,1,1,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[16395,1,2,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[16384,1,1,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[16395,1,2,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[16384,1,1,1],k=9999,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[16395,1,2,1],k=9999,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[16384,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[16395,1,2,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[16384,1,1,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[16395,1,2,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[16384,1,1,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[16395,1,2,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[16384,1,1,1],k=9999,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[16395,1,2,1],k=9999,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[32768,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[32779,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[32768,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -14014,14 +14500,14 @@ "SYCL0","TOP_K","type=f32,ne=[32779,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[32768,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[32779,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[32768,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[32779,1,2,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[32768,1,1,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[32779,1,2,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[32768,1,1,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[32779,1,2,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[32768,1,1,1],k=9999,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[32779,1,2,1],k=9999,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[32768,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[32779,1,2,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[32768,1,1,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[32779,1,2,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[32768,1,1,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[32779,1,2,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[32768,1,1,1],k=9999,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[32779,1,2,1],k=9999,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[65536,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[65547,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[65536,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -14032,14 +14518,14 @@ "SYCL0","TOP_K","type=f32,ne=[65547,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[65536,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[65547,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[65536,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[65547,1,2,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[65536,1,1,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[65547,1,2,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[65536,1,1,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[65547,1,2,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[65536,1,1,1],k=9999,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[65547,1,2,1],k=9999,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[65536,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[65547,1,2,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[65536,1,1,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[65547,1,2,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[65536,1,1,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[65547,1,2,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[65536,1,1,1],k=9999,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[65547,1,2,1],k=9999,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[131072,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[131083,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[131072,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -14050,14 +14536,14 @@ "SYCL0","TOP_K","type=f32,ne=[131083,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[131072,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[131083,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[131072,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[131083,1,2,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[131072,1,1,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[131083,1,2,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[131072,1,1,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[131083,1,2,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[131072,1,1,1],k=9999,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[131083,1,2,1],k=9999,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[131072,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[131083,1,2,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[131072,1,1,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[131083,1,2,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[131072,1,1,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[131083,1,2,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[131072,1,1,1],k=9999,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[131083,1,2,1],k=9999,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[262144,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[262155,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[262144,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -14068,14 +14554,14 @@ "SYCL0","TOP_K","type=f32,ne=[262155,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[262144,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[262155,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[262144,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[262155,1,2,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[262144,1,1,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[262155,1,2,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[262144,1,1,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[262155,1,2,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[262144,1,1,1],k=9999,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[262155,1,2,1],k=9999,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[262144,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[262155,1,2,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[262144,1,1,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[262155,1,2,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[262144,1,1,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[262155,1,2,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[262144,1,1,1],k=9999,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[262155,1,2,1],k=9999,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[524288,1,1,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[524299,1,2,1],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[524288,1,1,1],k=2,ties=0","support","1","yes","SYCL" @@ -14086,14 +14572,62 @@ "SYCL0","TOP_K","type=f32,ne=[524299,1,2,1],k=7,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[524288,1,1,1],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[524299,1,2,1],k=15,ties=0","support","1","yes","SYCL" -"SYCL0","TOP_K","type=f32,ne=[524288,1,1,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[524299,1,2,1],k=100,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[524288,1,1,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[524299,1,2,1],k=500,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[524288,1,1,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[524299,1,2,1],k=1023,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[524288,1,1,1],k=9999,ties=0","support","0","no","SYCL" -"SYCL0","TOP_K","type=f32,ne=[524299,1,2,1],k=9999,ties=0","support","0","no","SYCL" +"SYCL0","TOP_K","type=f32,ne=[524288,1,1,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[524299,1,2,1],k=100,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[524288,1,1,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[524299,1,2,1],k=500,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[524288,1,1,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[524299,1,2,1],k=1023,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[524288,1,1,1],k=9999,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[524299,1,2,1],k=9999,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,1,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,1,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,1,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,8,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,8,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,8,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,8,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,16,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,16,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,16,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,16,1,1],k=4,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,1,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,1,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,1,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,8,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,8,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,8,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,8,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,16,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,16,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,16,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,16,1,1],k=8,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,1,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,1,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,1,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,8,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,8,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,8,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,8,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,16,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,16,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,16,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,16,1,1],k=16,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,1,1,1],k=32,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,1,1,1],k=32,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,1,1,1],k=32,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,1,1,1],k=32,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,8,1,1],k=32,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,8,1,1],k=32,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,8,1,1],k=32,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,8,1,1],k=32,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[202048,16,1,1],k=32,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[151936,16,1,1],k=32,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,16,1,1],k=32,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8193,16,1,1],k=32,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[16,10,10,10],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[60,10,10,10],k=1,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[1023,2,1,3],k=1,ties=0","support","1","yes","SYCL" @@ -14139,6 +14673,12 @@ "SYCL0","TOP_K","type=f32,ne=[2047,2,1,3],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[2048,2,1,3],k=15,ties=0","support","1","yes","SYCL" "SYCL0","TOP_K","type=f32,ne=[2049,2,1,3],k=15,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[1024,1,1,1],k=1024,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[2048,2,1,1],k=1024,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[4096,1,1,1],k=2048,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[8192,2,1,1],k=2051,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[33024,1,1,1],k=2051,ties=0","support","1","yes","SYCL" +"SYCL0","TOP_K","type=f32,ne=[33024,4,1,1],k=2051,ties=0","support","1","yes","SYCL" "SYCL0","UPSCALE","type=f32,ne=[512,512,3,2],scale_factor=2,mode=nearest,transpose=0","support","1","yes","SYCL" "SYCL0","UPSCALE","type=f32,ne=[512,512,3,2],scale_factor=2,mode=nearest,transpose=1","support","1","yes","SYCL" "SYCL0","UPSCALE","type=f32,ne=[2,5,7,11],ne_tgt=[5,7,11,13],mode=nearest","support","1","yes","SYCL" @@ -14165,13 +14705,16 @@ "SYCL0","SUM","type=f32,ne=[11,5,6,3],permute=[0,2,1,3]","support","0","no","SYCL" "SYCL0","SUM","type=f32,ne=[11,5,6,3],permute=[0,3,2,1]","support","0","no","SYCL" "SYCL0","SUM","type=f32,ne=[11,5,6,3],permute=[0,1,3,2]","support","0","no","SYCL" -"SYCL0","MEAN","type=f32,ne=[10,5,4,3]","support","1","yes","SYCL" -"SYCL0","MEAN","type=f32,ne=[33,1,1,1]","support","1","yes","SYCL" -"SYCL0","MEAN","type=f32,ne=[33,256,1,1]","support","1","yes","SYCL" -"SYCL0","MEAN","type=f32,ne=[32769,1,1,1]","support","1","yes","SYCL" -"SYCL0","MEAN","type=f32,ne=[32,1,1,1]","support","1","yes","SYCL" -"SYCL0","MEAN","type=f32,ne=[32,256,1,1]","support","1","yes","SYCL" -"SYCL0","MEAN","type=f32,ne=[32768,1,1,1]","support","1","yes","SYCL" +"SYCL0","MEAN","type=f32,ne=[10,5,4,3],permute=0,slice=0","support","1","yes","SYCL" +"SYCL0","MEAN","type=f32,ne=[33,1,1,1],permute=0,slice=0","support","1","yes","SYCL" +"SYCL0","MEAN","type=f32,ne=[33,256,1,1],permute=0,slice=0","support","1","yes","SYCL" +"SYCL0","MEAN","type=f32,ne=[32769,1,1,1],permute=0,slice=0","support","1","yes","SYCL" +"SYCL0","MEAN","type=f32,ne=[32,1,1,1],permute=0,slice=0","support","1","yes","SYCL" +"SYCL0","MEAN","type=f32,ne=[32,256,1,1],permute=0,slice=0","support","1","yes","SYCL" +"SYCL0","MEAN","type=f32,ne=[32768,1,1,1],permute=0,slice=0","support","1","yes","SYCL" +"SYCL0","MEAN","type=f32,ne=[11,5,6,3],permute=1,slice=0","support","0","no","SYCL" +"SYCL0","MEAN","type=f32,ne=[11,5,6,3],permute=0,slice=1","support","0","no","SYCL" +"SYCL0","MEAN","type=f32,ne=[11,5,6,3],permute=1,slice=1","support","0","no","SYCL" "SYCL0","SUM","type=f32,ne=[33,1,1,1]","support","1","yes","SYCL" "SYCL0","SUM","type=f32,ne=[33,1024,1,1]","support","1","yes","SYCL" "SYCL0","SUM","type=f32,ne=[33,256,1,1]","support","1","yes","SYCL" @@ -14284,5147 +14827,5311 @@ "SYCL0","PAD","type=f32,ne_a=[11,22,33,44],lp0=1,rp0=2,lp1=3,rp1=4,lp2=5,rp2=6,lp3=7,rp3=8,tfrm=2,circular=0","support","1","yes","SYCL" "SYCL0","PAD","type=f32,ne_a=[512,512,1,1],lp0=0,rp0=1,lp1=0,rp1=1,lp2=0,rp2=0,lp3=0,rp3=0,tfrm=2,circular=1","support","0","no","SYCL" "SYCL0","PAD","type=f32,ne_a=[11,22,33,44],lp0=1,rp0=2,lp1=3,rp1=4,lp2=5,rp2=6,lp3=7,rp3=8,tfrm=2,circular=1","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=96,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=96,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f32,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=256,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=96,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q1_0,type_V=q1_0,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q1_0,type_V=q4_0,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=128,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q1_0,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=64,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q1_0,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=96,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q2_0,type_V=q2_0,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q2_0,type_V=q4_0,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=128,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q2_0,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=64,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q2_0,type_V=f16,permute=[0,1,2,3]","support","0","no","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[6,1],kv=4096,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=4096,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[6,1],kv=16384,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" -"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=16384,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_1,type_V=q5_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=96,hsv=96,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[12,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=10.000000,prec=def,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[16,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[8,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=192,nh=4,nr23=[16,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=64,nh=4,nr23=[1,1],kv=256,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=72,nh=4,nr23=[1,1],kv=256,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=96,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=96,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f32,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=256,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=96,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q1_0,type_V=q1_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q1_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=128,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q1_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=64,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q1_0,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=96,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q2_0,type_V=q2_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q2_0,type_V=q4_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=128,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q2_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=64,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q2_0,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=2,nr23=[16,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=2,nr23=[16,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=2,nr23=[16,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=2,nr23=[16,2],kv=1025,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=30.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=2,nr23=[16,1],kv=1025,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=2,nr23=[16,1],kv=16384,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[8,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[8,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[8,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=1,nr23=[8,1],kv=4096,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=1,nr23=[8,2],kv=4096,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=768","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[16,1],kv=4096,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[16,2],kv=4096,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=768","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=1,nr23=[8,1],kv=4096,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=2304","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=1,nr23=[8,1],kv=4096,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=1,nr23=[8,1],kv=4096,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=1,nr23=[8,1],kv=4096,nb=4,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=1,nr23=[8,1],kv=4096,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=1,nr23=[8,1],kv=4096,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=2048","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[16,1],kv=4096,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[16,1],kv=4096,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=2048","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=1,nr23=[8,1],kv=4096,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=1,nr23=[8,1],kv=4096,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=1,nr23=[8,1],kv=4096,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=2048","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=2,nr23=[12,1],kv=4096,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=2,nr23=[12,1],kv=8192,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=1,nr23=[12,2],kv=8192,nb=67,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=1,nr23=[8,1],kv=4096,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=1,nr23=[8,1],kv=4096,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=512","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=2,nr23=[12,1],kv=8192,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=2048","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=320,hsv=256,nh=1,nr23=[32,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=4,nr23=[8,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=512,nb=8,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[6,1],kv=4096,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=4096,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[6,1],kv=16384,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=16384,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[6,1],kv=512,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[6,1],kv=4096,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[6,1],kv=4096,nb=16,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[2,1],kv=4096,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[4,1],kv=4096,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=2,nr23=[12,1],kv=4096,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=0,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=0,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,2,1,3],kv_view=0,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3],kv_view=0,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=192,hsv=128,nh=8,nr23=[8,1],kv=4096,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","0","no","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=576,hsv=512,nh=1,nr23=[20,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=1,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[8,1],kv=4096,nb=4,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=4096,nb=8,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[2,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" +"SYCL0","FLASH_ATTN_EXT","hsk=512,hsv=512,nh=4,nr23=[2,1],kv=1024,nb=4,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3],kv_view=1,v_is_view_of_k=0,n_kv_max=0","support","1","yes","SYCL" "SYCL0","CROSS_ENTROPY_LOSS","type=f32,ne=[10,5,4,3]","support","1","yes","SYCL" "SYCL0","CROSS_ENTROPY_LOSS","type=f32,ne=[30000,1,1,1]","support","1","yes","SYCL" "SYCL0","CROSS_ENTROPY_LOSS_BACK","type=f32,ne=[10,5,4,3]","support","1","yes","SYCL" diff --git a/examples/convert-llama2c-to-ggml/CMakeLists.txt b/examples/convert-llama2c-to-ggml/CMakeLists.txt index 2162da4fdf76..d921af9a495d 100644 --- a/examples/convert-llama2c-to-ggml/CMakeLists.txt +++ b/examples/convert-llama2c-to-ggml/CMakeLists.txt @@ -1,5 +1,7 @@ -set(TARGET llama-convert-llama2c-to-ggml) -add_executable(${TARGET} convert-llama2c-to-ggml.cpp) -install(TARGETS ${TARGET} RUNTIME) -target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT}) -target_compile_features(${TARGET} PRIVATE cxx_std_17) +if (GGML_CPU) + set(TARGET llama-convert-llama2c-to-ggml) + add_executable(${TARGET} convert-llama2c-to-ggml.cpp) + install(TARGETS ${TARGET} RUNTIME) + target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT}) + target_compile_features(${TARGET} PRIVATE cxx_std_17) +endif() diff --git a/examples/eval-callback/CMakeLists.txt b/examples/eval-callback/CMakeLists.txt index 63fbe59dce87..96e1e1b35327 100644 --- a/examples/eval-callback/CMakeLists.txt +++ b/examples/eval-callback/CMakeLists.txt @@ -18,7 +18,7 @@ if(LLAMA_BUILD_TESTS) -DDEST=${MODEL_DEST} -DNAME=${MODEL_NAME} -DHASH=${MODEL_HASH} - -P ${CMAKE_SOURCE_DIR}/cmake/download-models.cmake + -P ${PROJECT_SOURCE_DIR}/cmake/download-models.cmake ) set_tests_properties(${TEST_TARGET}-download-model PROPERTIES FIXTURES_SETUP ${TEST_TARGET}-download-model) add_test(NAME ${TEST_TARGET} COMMAND llama-eval-callback -m "${MODEL_DEST}" --prompt hello --seed 42 -ngl 0) diff --git a/examples/json_schema_to_grammar.py b/examples/json_schema_to_grammar.py deleted file mode 100755 index 02b7ef15ae98..000000000000 --- a/examples/json_schema_to_grammar.py +++ /dev/null @@ -1,842 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import itertools -import json -import re -import sys -from typing import Any, List, Optional, Set, Tuple, Union - -def _build_repetition(item_rule, min_items, max_items, separator_rule=None): - - if max_items == 0: - return "" - - if min_items == 0 and max_items == 1: - return f'{item_rule}?' - - if not separator_rule: - if min_items == 1 and max_items is None: - return f'{item_rule}+' - elif min_items == 0 and max_items is None: - return f'{item_rule}*' - else: - return f'{item_rule}{{{min_items},{max_items if max_items is not None else ""}}}' - - result = item_rule + ' ' + _build_repetition(f'({separator_rule} {item_rule})', min_items - 1 if min_items > 0 else 0, max_items - 1 if max_items is not None else None) - return f'({result})?' if min_items == 0 else result - -def _generate_min_max_int(min_value: Optional[int], max_value: Optional[int], out: list, decimals_left: int = 16, top_level: bool = True): - def digit_range(from_char: str, to_char: str): - out.append("[") - if from_char == to_char: - out.append(from_char) - else: - out.append(from_char) - out.append("-") - out.append(to_char) - out.append("]") - - def more_digits(min_digits: int, max_digits: int): - out.append("[0-9]") - if min_digits == max_digits and min_digits == 1: - return - out.append("{") - out.append(str(min_digits)) - if max_digits != min_digits: - out.append(",") - if max_digits != sys.maxsize: - out.append(str(max_digits)) - out.append("}") - - def uniform_range(from_str: str, to_str: str): - i = 0 - while i < len(from_str) and from_str[i] == to_str[i]: - i += 1 - if i > 0: - out.append("\"") - out.append(from_str[:i]) - out.append("\"") - if i < len(from_str): - if i > 0: - out.append(" ") - sub_len = len(from_str) - i - 1 - if sub_len > 0: - from_sub = from_str[i+1:] - to_sub = to_str[i+1:] - sub_zeros = "0" * sub_len - sub_nines = "9" * sub_len - - to_reached = False - out.append("(") - if from_sub == sub_zeros: - digit_range(from_str[i], chr(ord(to_str[i]) - 1)) - out.append(" ") - more_digits(sub_len, sub_len) - else: - out.append("[") - out.append(from_str[i]) - out.append("] ") - out.append("(") - uniform_range(from_sub, sub_nines) - out.append(")") - if ord(from_str[i]) < ord(to_str[i]) - 1: - out.append(" | ") - if to_sub == sub_nines: - digit_range(chr(ord(from_str[i]) + 1), to_str[i]) - to_reached = True - else: - digit_range(chr(ord(from_str[i]) + 1), chr(ord(to_str[i]) - 1)) - out.append(" ") - more_digits(sub_len, sub_len) - if not to_reached: - out.append(" | ") - digit_range(to_str[i], to_str[i]) - out.append(" ") - uniform_range(sub_zeros, to_sub) - out.append(")") - else: - out.append("[") - out.append(from_str[i]) - out.append("-") - out.append(to_str[i]) - out.append("]") - - if min_value is not None and max_value is not None: - if min_value < 0 and max_value < 0: - out.append("\"-\" (") - _generate_min_max_int(-max_value, -min_value, out, decimals_left, top_level=True) - out.append(")") - return - - if min_value < 0: - out.append("\"-\" (") - _generate_min_max_int(0, -min_value, out, decimals_left, top_level=True) - out.append(") | ") - min_value = 0 - - min_s = str(min_value) - max_s = str(max_value) - min_digits = len(min_s) - max_digits = len(max_s) - - for digits in range(min_digits, max_digits): - uniform_range(min_s, "9" * digits) - min_s = "1" + "0" * digits - out.append(" | ") - uniform_range(min_s, max_s) - return - - less_decimals = max(decimals_left - 1, 1) - - if min_value is not None: - if min_value < 0: - out.append("\"-\" (") - _generate_min_max_int(None, -min_value, out, decimals_left, top_level=False) - out.append(") | [0] | [1-9] ") - more_digits(0, decimals_left - 1) - elif min_value == 0: - if top_level: - out.append("[0] | [1-9] ") - more_digits(0, less_decimals) - else: - more_digits(1, decimals_left) - elif min_value <= 9: - c = str(min_value) - range_start = '1' if top_level else '0' - if c > range_start: - digit_range(range_start, chr(ord(c) - 1)) - out.append(" ") - more_digits(1, less_decimals) - out.append(" | ") - digit_range(c, "9") - out.append(" ") - more_digits(0, less_decimals) - else: - min_s = str(min_value) - length = len(min_s) - c = min_s[0] - - if c > "1": - digit_range("1" if top_level else "0", chr(ord(c) - 1)) - out.append(" ") - more_digits(length, less_decimals) - out.append(" | ") - digit_range(c, c) - out.append(" (") - _generate_min_max_int(int(min_s[1:]), None, out, less_decimals, top_level=False) - out.append(")") - if c < "9": - out.append(" | ") - digit_range(chr(ord(c) + 1), "9") - out.append(" ") - more_digits(length - 1, less_decimals) - return - - if max_value is not None: - if max_value >= 0: - if top_level: - out.append("\"-\" [1-9] ") - more_digits(0, less_decimals) - out.append(" | ") - _generate_min_max_int(0, max_value, out, decimals_left, top_level=True) - else: - out.append("\"-\" (") - _generate_min_max_int(-max_value, None, out, decimals_left, top_level=False) - out.append(")") - return - - raise RuntimeError("At least one of min_value or max_value must be set") - -class BuiltinRule: - def __init__(self, content: str, deps: list | None = None): - self.content = content - self.deps = deps or [] - -# Constraining spaces to prevent model "running away". -SPACE_RULE = '| " " | "\\n"{1,2} [ \\t]{0,20}' - -PRIMITIVE_RULES = { - 'boolean' : BuiltinRule('("true" | "false")', []), - 'decimal-part' : BuiltinRule('[0-9]{1,16}', []), - 'integral-part': BuiltinRule('[0] | [1-9] [0-9]{0,15}', []), - 'number' : BuiltinRule('("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)?', ['integral-part', 'decimal-part']), - 'integer' : BuiltinRule('("-"? integral-part)', ['integral-part']), - 'value' : BuiltinRule('object | array | string | number | boolean | null', ['object', 'array', 'string', 'number', 'boolean', 'null']), - 'object' : BuiltinRule('"{" space ( string ":" space value ("," space string ":" space value)* )? space "}"', ['string', 'value']), - 'array' : BuiltinRule('"[" space ( value ("," space value)* )? space "]"', ['value']), - 'uuid' : BuiltinRule(r'"\"" [0-9a-fA-F]{8} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{12} "\""', []), - 'char' : BuiltinRule(r'[^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})', []), - 'string' : BuiltinRule(r'"\"" char* "\""', ['char']), - 'null' : BuiltinRule('"null"', []), -} - -# TODO: support "uri", "email" string formats -STRING_FORMAT_RULES = { - 'date' : BuiltinRule('[0-9]{4} "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', []), - 'time' : BuiltinRule('([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9]{3} )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', []), - 'date-time' : BuiltinRule('date "T" time', ['date', 'time']), - 'date-string' : BuiltinRule('"\\"" date "\\""', ['date']), - 'time-string' : BuiltinRule('"\\"" time "\\""', ['time']), - 'date-time-string': BuiltinRule('"\\"" date-time "\\""', ['date-time']), -} - -DOTALL = '[\\U00000000-\\U0010FFFF]' -DOT = '[^\\x0A\\x0D]' - -RESERVED_NAMES = set(["root", "dot", *PRIMITIVE_RULES.keys(), *STRING_FORMAT_RULES.keys()]) - -INVALID_RULE_CHARS_RE = re.compile(r'[^a-zA-Z0-9-]+') -GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\\]') -GRAMMAR_RANGE_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\]\-\\]') -GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"', '-': '\\-', ']': '\\]', '\\': '\\\\'} - -NON_LITERAL_SET = set('|.()[]{}*+?') -ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('^$.[]()|{}*+?') - - -class SchemaConverter: - def __init__(self, *, prop_order, allow_fetch, dotall, raw_pattern): - self._prop_order = prop_order - self._allow_fetch = allow_fetch - self._dotall = dotall - self._raw_pattern = raw_pattern - self._rules = { - 'space': SPACE_RULE, - } - self._refs = {} - self._refs_being_resolved = set() - - def _format_literal(self, literal): - escaped = GRAMMAR_LITERAL_ESCAPE_RE.sub( - lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)) or m.group(0), literal - ) - return f'"{escaped}"' - - def not_literal(self, literal: str, dotall: bool = True, maybe_escaped_underscores = False) -> str: - ''' - not_literal('a') -> '[^a]' - not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?' - ''' - assert len(literal) > 0, 'Empty literal not supported' - def recurse(i: int): - c = literal[i] - if maybe_escaped_underscores and c == '_': - yield f'[^{c}\\\\]' - yield ' | ' - yield f'"\\\\"? "{c}"' - else: - yield f'[^{c}]' - if i < len(literal) - 1: - yield ' | ' - yield self._format_literal(c) - yield ' (' - yield from recurse(i + 1) - yield ')?' - - return ''.join(('(', *recurse(0), ')')) - - def _not_strings(self, strings): - class TrieNode: - def __init__(self): - self.children = {} - self.is_end_of_string = False - - def insert(self, string): - node = self - for c in string: - node = node.children.setdefault(c, TrieNode()) - node.is_end_of_string = True - - trie = TrieNode() - for s in strings: - trie.insert(s) - - char_rule = self._add_primitive('char', PRIMITIVE_RULES['char']) - out = ['["] ( '] - - def visit(node): - rejects = [] - first = True - for c in sorted(node.children.keys()): - child = node.children[c] - rejects.append(c) - if first: - first = False - else: - out.append(' | ') - out.append(f'[{c}]') - if child.children: - out.append(f' (') - visit(child) - out.append(')') - elif child.is_end_of_string: - out.append(f' {char_rule}+') - if node.children: - if not first: - out.append(' | ') - out.append(f'[^"{"".join(rejects)}] {char_rule}*') - visit(trie) - - out.append(f' ){"" if trie.is_end_of_string else "?"} ["]') - return ''.join(out) - - def _add_rule(self, name, rule): - esc_name = INVALID_RULE_CHARS_RE.sub('-', name) - if esc_name not in self._rules or self._rules[esc_name] == rule: - key = esc_name - else: - i = 0 - while f'{esc_name}{i}' in self._rules and self._rules[f'{esc_name}{i}'] != rule: - i += 1 - key = f'{esc_name}{i}' - self._rules[key] = rule - return key - - def resolve_refs(self, schema: dict, url: str): - ''' - Resolves all $ref fields in the given schema, fetching any remote schemas, - replacing $ref with absolute reference URL and populating self._refs with the - respective referenced (sub)schema dictionaries. - ''' - def visit(n: dict): - if isinstance(n, list): - return [visit(x) for x in n] - elif isinstance(n, dict): - ref = n.get('$ref') - if ref is not None and ref not in self._refs: - if ref.startswith('https://'): - assert self._allow_fetch, 'Fetching remote schemas is not allowed (use --allow-fetch for force)' - import requests - - frag_split = ref.split('#') - base_url = frag_split[0] - - target = self._refs.get(base_url) - if target is None: - target = self.resolve_refs(requests.get(ref).json(), base_url) - self._refs[base_url] = target - - if len(frag_split) == 1 or frag_split[-1] == '': - return target - elif ref.startswith('#/'): - target = schema - ref = f'{url}{ref}' - n['$ref'] = ref - else: - raise ValueError(f'Unsupported ref {ref}') - - for sel in ref.split('#')[-1].split('/')[1:]: - assert target is not None, f'Error resolving ref {ref}: {sel} not in {target}' - if isinstance(target, list): - try: - sel_index = int(sel) - except ValueError: - raise ValueError(f'Error resolving ref {ref}: {sel} not in {target}') - assert 0 <= sel_index < len(target), f'Error resolving ref {ref}: {sel} not in {target}' - target = target[sel_index] - else: - assert sel in target, f'Error resolving ref {ref}: {sel} not in {target}' - target = target[sel] - - self._refs[ref] = target - else: - for v in n.values(): - visit(v) - - return n - return visit(schema) - - def _generate_union_rule(self, name, alt_schemas): - return ' | '.join(( - self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}') - for i, alt_schema in enumerate(alt_schemas) - )) - - def _visit_pattern(self, pattern, name): - ''' - Transforms a regular expression pattern into a GBNF rule. - - Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions - Output: https://github.com/ggml-org/llama.cpp/blob/master/grammars/README.md - - Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. - - Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which - we define sub-rules to keep the output lean. - ''' - - assert pattern.startswith('^') and pattern.endswith('$'), 'Pattern must start with "^" and end with "$"' - pattern = pattern[1:-1] - sub_rule_ids = {} - - i = 0 - length = len(pattern) - - def to_rule(s: tuple[str, bool]) -> str: - (txt, is_literal) = s - return "\"" + txt + "\"" if is_literal else txt - - def transform() -> tuple[str, bool]: - ''' - Parse a unit at index i (advancing it), and return its string representation + whether it's a literal. - ''' - nonlocal i - nonlocal pattern - nonlocal sub_rule_ids - - start = i - # For each component of this sequence, store its string representation and whether it's a literal. - # We only need a flat structure here to apply repetition operators to the last item, and - # to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially - # (GBNF's syntax is luckily very close to regular expressions!) - seq: list[tuple[str, bool]] = [] - - def get_dot(): - if self._dotall: - rule = DOTALL - else: - # Accept any character... except \n and \r line break chars (\x0A and \xOD) - rule = DOT - return self._add_rule(f'dot', rule) - - def join_seq(): - nonlocal seq - ret = [] - for is_literal, g in itertools.groupby(seq, lambda x: x[1]): - if is_literal: - ret.append((''.join(x[0] for x in g), True)) - else: - ret.extend(g) - if len(ret) == 1: - return ret[0] - return (' '.join(to_rule(x) for x in seq), False) - - while i < length: - c = pattern[i] - if c == '.': - seq.append((get_dot(), False)) - i += 1 - elif c == '(': - i += 1 - if i < length: - assert pattern[i] != '?', f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/' - seq.append((f'({to_rule(transform())})', False)) - elif c == ')': - i += 1 - assert start > 0 and pattern[start-1] == '(', f'Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}' - return join_seq() - elif c == '[': - square_brackets = c - i += 1 - while i < length and pattern[i] != ']': - if pattern[i] == '\\': - square_brackets += pattern[i:i+2] - i += 2 - else: - square_brackets += pattern[i] - i += 1 - assert i < length, f'Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}' - square_brackets += ']' - i += 1 - seq.append((square_brackets, False)) - elif c == '|': - seq.append(('|', False)) - i += 1 - elif c in ('*', '+', '?'): - seq[-1] = (to_rule(seq[-1]) + c, False) - i += 1 - elif c == '{': - curly_brackets = c - i += 1 - while i < length and pattern[i] != '}': - curly_brackets += pattern[i] - i += 1 - assert i < length, f'Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}' - curly_brackets += '}' - i += 1 - nums = [s.strip() for s in curly_brackets[1:-1].split(',')] - min_times = 0 - max_times = None - try: - if len(nums) == 1: - min_times = int(nums[0]) - max_times = min_times - else: - assert len(nums) == 2 - min_times = int(nums[0]) if nums[0] else 0 - max_times = int(nums[1]) if nums[1] else None - except ValueError: - raise ValueError(f'Invalid quantifier {curly_brackets} in /{pattern}/') - - (sub, sub_is_literal) = seq[-1] - - if not sub_is_literal: - id = sub_rule_ids.get(sub) - if id is None: - id = self._add_rule(f'{name}-{len(sub_rule_ids) + 1}', sub) - sub_rule_ids[sub] = id - sub = id - - seq[-1] = (_build_repetition(f'"{sub}"' if sub_is_literal else sub, min_times, max_times), False) - else: - literal = '' - while i < length: - if pattern[i] == '\\' and i < length - 1: - next = pattern[i + 1] - if next in ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS: - i += 1 - literal += pattern[i] - i += 1 - else: - literal += pattern[i:i+2] - i += 2 - elif pattern[i] == '"' and not self._raw_pattern: - literal += '\\"' - i += 1 - elif pattern[i] not in NON_LITERAL_SET and \ - (i == length - 1 or literal == '' or pattern[i+1] == '.' or pattern[i+1] not in NON_LITERAL_SET): - literal += pattern[i] - i += 1 - else: - break - if literal: - seq.append((literal, True)) - - return join_seq() - - return self._add_rule( - name, - to_rule(transform()) if self._raw_pattern \ - else "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\"") - - - def _resolve_ref(self, ref): - ref_fragment = ref.split('#')[-1] - ref_name = 'ref' + re.sub(r'[^a-zA-Z0-9-]+', '-', ref_fragment) - if ref_name not in self._rules and ref not in self._refs_being_resolved: - self._refs_being_resolved.add(ref) - resolved = self._refs[ref] - ref_name = self.visit(resolved, ref_name) - self._refs_being_resolved.remove(ref) - return ref_name - - def _generate_constant_rule(self, value): - return self._format_literal(json.dumps(value)) - - def visit(self, schema, name): - schema_type = schema.get('type') - schema_format = schema.get('format') - rule_name = name + '-' if name in RESERVED_NAMES else name or 'root' - - if (ref := schema.get('$ref')) is not None: - return self._add_rule(rule_name, self._resolve_ref(ref)) - - elif 'oneOf' in schema or 'anyOf' in schema: - return self._add_rule(rule_name, self._generate_union_rule(name, schema.get('oneOf') or schema['anyOf'])) - - elif isinstance(schema_type, list): - return self._add_rule(rule_name, self._generate_union_rule(name, [{**schema, 'type': t} for t in schema_type])) - - elif 'const' in schema: - return self._add_rule(rule_name, self._generate_constant_rule(schema['const'])) - - elif 'enum' in schema: - rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) + ')' - return self._add_rule(rule_name, rule) - - elif schema_type in (None, 'object') and \ - ('properties' in schema or \ - ('additionalProperties' in schema and schema['additionalProperties'] is not True)): - required = set(schema.get('required', [])) - properties = list(schema.get('properties', {}).items()) - return self._add_rule(rule_name, self._build_object_rule(properties, required, name, schema.get('additionalProperties'))) - - elif schema_type in (None, 'object', 'string') and 'allOf' in schema: - required = set() - properties = [] - enum_sets = [] - hybrid_name = name - def add_component(comp_schema, is_required): - if (ref := comp_schema.get('$ref')) is not None: - comp_schema = self._refs[ref] - - if 'properties' in comp_schema: - for prop_name, prop_schema in comp_schema['properties'].items(): - properties.append((prop_name, prop_schema)) - if is_required: - required.add(prop_name) - - if 'enum' in comp_schema: - enum_sets.append(set(comp_schema['enum'])) - - for t in schema['allOf']: - if 'anyOf' in t: - for tt in t['anyOf']: - add_component(tt, is_required=False) - else: - add_component(t, is_required=True) - - if enum_sets: - enum_intersection = enum_sets[0] - for s in enum_sets[1:]: - enum_intersection &= s - - if enum_intersection: - rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in sorted(enum_intersection))) + ')' - return self._add_rule(rule_name, rule) - - return self._add_rule(rule_name, self._build_object_rule(properties, required, hybrid_name, additional_properties=None)) - - elif schema_type in (None, 'array') and ('items' in schema or 'prefixItems' in schema): - items = schema.get('items', schema.get('prefixItems')) - if isinstance(items, list): - return self._add_rule( - rule_name, - '"[" space ' + - ' "," space '.join( - self.visit(item, f'{name}{"-" if name else ""}tuple-{i}') - for i, item in enumerate(items)) + - ' space "]"') - else: - item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item') - min_items = schema.get("minItems", 0) - max_items = schema.get("maxItems") - return self._add_rule(rule_name, '"[" space ' + _build_repetition(item_rule_name, min_items, max_items, separator_rule='"," space') + ' space "]"') - - elif schema_type in (None, 'string') and 'pattern' in schema: - return self._visit_pattern(schema['pattern'], rule_name) - - elif schema_type in (None, 'string') and re.match(r'^uuid[1-5]?$', schema_format or ''): - return self._add_primitive( - 'root' if rule_name == 'root' else schema_format, - PRIMITIVE_RULES['uuid'] - ) - - elif schema_type in (None, 'string') and f'{schema_format}-string' in STRING_FORMAT_RULES: - prim_name = f'{schema_format}-string' - return self._add_rule(rule_name, self._add_primitive(prim_name, STRING_FORMAT_RULES[prim_name])) - - elif schema_type == 'string' and ('minLength' in schema or 'maxLength' in schema): - char_rule = self._add_primitive('char', PRIMITIVE_RULES['char']) - min_len = schema.get('minLength', 0) - max_len = schema.get('maxLength') - - return self._add_rule(rule_name, r'"\"" ' + _build_repetition(char_rule, min_len, max_len) + r' "\""') - - elif schema_type in (None, 'integer') and \ - ('minimum' in schema or 'exclusiveMinimum' in schema or 'maximum' in schema or 'exclusiveMaximum' in schema): - min_value = None - max_value = None - if 'minimum' in schema: - min_value = schema['minimum'] - elif 'exclusiveMinimum' in schema: - min_value = schema['exclusiveMinimum'] + 1 - if 'maximum' in schema: - max_value = schema['maximum'] - elif 'exclusiveMaximum' in schema: - max_value = schema['exclusiveMaximum'] - 1 - - out = ["("] - _generate_min_max_int(min_value, max_value, out) - out.append(")") - return self._add_rule(rule_name, ''.join(out)) - - elif (schema_type == 'object') or (len(schema) == 0): - return self._add_rule(rule_name, self._add_primitive('object', PRIMITIVE_RULES['object'])) - - elif schema_type is None and isinstance(schema, dict): - # No type constraint and no recognized structural keywords (e.g. {"description": "..."}). - # Per JSON Schema semantics this is equivalent to {} and accepts any value. - return self._add_rule(rule_name, self._add_primitive('value', PRIMITIVE_RULES['value'])) - - else: - assert schema_type in PRIMITIVE_RULES, f'Unrecognized schema: {schema}' - # TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero - return self._add_primitive('root' if rule_name == 'root' else schema_type, PRIMITIVE_RULES[schema_type]) - - def _add_primitive(self, name: str, rule: BuiltinRule): - n = self._add_rule(name, rule.content) - - for dep in rule.deps: - dep_rule = PRIMITIVE_RULES.get(dep) or STRING_FORMAT_RULES.get(dep) - assert dep_rule, f'Rule {dep} not known' - if dep not in self._rules: - self._add_primitive(dep, dep_rule) - return n - - def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[str], name: str, additional_properties: Optional[Union[bool, Any]]): - prop_order = self._prop_order - # sort by position in prop_order (if specified) then by original order - sorted_props = [kv[0] for _, kv in sorted(enumerate(properties), key=lambda ikv: (prop_order.get(ikv[1][0], len(prop_order)), ikv[0]))] - - prop_kv_rule_names = {} - for prop_name, prop_schema in properties: - prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}') - prop_kv_rule_names[prop_name] = self._add_rule( - f'{name}{"-" if name else ""}{prop_name}-kv', - fr'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}' - ) - required_props = [k for k in sorted_props if k in required] - optional_props = [k for k in sorted_props if k not in required] - - if additional_properties is not None and additional_properties != False: - sub_name = f'{name}{"-" if name else ""}additional' - value_rule = self.visit(additional_properties, f'{sub_name}-value') if isinstance(additional_properties, dict) else \ - self._add_primitive('value', PRIMITIVE_RULES['value']) - key_rule = self._add_primitive('string', PRIMITIVE_RULES['string']) if not sorted_props \ - else self._add_rule(f'{sub_name}-k', self._not_strings(sorted_props)) - - prop_kv_rule_names["*"] = self._add_rule( - f'{sub_name}-kv', - f'{key_rule} ":" space {value_rule}' - ) - optional_props.append("*") - - if not required_props and not optional_props: - return '"{" space "}"' - - rule = '"{" space ' - rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props) - - if optional_props: - rule += ' (' - if required_props: - rule += ' "," space ( ' - - def get_recursive_refs(ks, first_is_optional): - [k, *rest] = ks - kv_rule_name = prop_kv_rule_names[k] - comma_ref = f'( "," space {kv_rule_name} )' - if first_is_optional: - res = comma_ref + ('*' if k == '*' else '?') - else: - res = kv_rule_name + (' ' + comma_ref + "*" if k == '*' else '') - if len(rest) > 0: - res += ' ' + self._add_rule( - f'{name}{"-" if name else ""}{k}-rest', - get_recursive_refs(rest, first_is_optional=True) - ) - return res - - rule += ' | '.join( - get_recursive_refs(optional_props[i:], first_is_optional=False) - for i in range(len(optional_props)) - ) - if required_props: - rule += ' )' - rule += ' )?' - - rule += ' space "}"' - - return rule - - def format_grammar(self): - return '\n'.join( - f'{name} ::= {rule}' - for name, rule in sorted(self._rules.items(), key=lambda kv: kv[0]) - ) - - -def main(args_in = None): - parser = argparse.ArgumentParser( - description=''' - Generates a grammar (suitable for use in ./llama-cli) that produces JSON conforming to a - given JSON schema. Only a subset of JSON schema features are supported; more may be - added in the future. - ''', - ) - parser.add_argument( - '--prop-order', - default=[], - type=lambda s: s.split(','), - help=''' - comma-separated property names defining the order of precedence for object properties; - properties not specified here are given lower precedence than those that are, and - are kept in their original order from the schema. Required properties are always - given precedence over optional properties. - ''' - ) - parser.add_argument( - '--allow-fetch', - action='store_true', - default=False, - help='Whether to allow fetching referenced schemas over HTTPS') - parser.add_argument( - '--dotall', - action='store_true', - default=False, - help='Whether to treat dot (".") as matching all chars including line breaks in regular expression patterns') - parser.add_argument( - '--raw-pattern', - action='store_true', - default=False, - help='Treats string patterns as raw patterns w/o quotes (or quote escapes)') - - parser.add_argument('schema', help='file containing JSON schema ("-" for stdin)') - args = parser.parse_args(args_in) - - if args.schema.startswith('https://'): - url = args.schema - import requests - schema = requests.get(url).json() - elif args.schema == '-': - url = 'stdin' - schema = json.load(sys.stdin) - else: - url = f'file://{args.schema}' - with open(args.schema) as f: - schema = json.load(f) - converter = SchemaConverter( - prop_order={name: idx for idx, name in enumerate(args.prop_order)}, - allow_fetch=args.allow_fetch, - dotall=args.dotall, - raw_pattern=args.raw_pattern) - schema = converter.resolve_refs(schema, url) - converter.visit(schema, '') - print(converter.format_grammar()) - - -if __name__ == '__main__': - main() diff --git a/examples/model-conversion/Makefile b/examples/model-conversion/Makefile index 0130b0493bb2..b80dc0f411f4 100644 --- a/examples/model-conversion/Makefile +++ b/examples/model-conversion/Makefile @@ -68,6 +68,9 @@ causal-run-converted-model: @CONVERTED_MODEL="$(CONVERTED_MODEL)" ./scripts/causal/run-converted-model.sh causal-verify-logits: causal-run-original-model causal-run-converted-model + $(MAKE) causal-compare-logits + +causal-compare-logits: @MODEL_PATH="$(MODEL_PATH)" ./scripts/causal/compare-logits.py @MODEL_PATH="$(MODEL_PATH)" ./scripts/utils/check-nmse.py -m ${MODEL_PATH} diff --git a/examples/parallel/parallel.cpp b/examples/parallel/parallel.cpp index a46400c5b943..4b74540f0707 100644 --- a/examples/parallel/parallel.cpp +++ b/examples/parallel/parallel.cpp @@ -11,6 +11,7 @@ #include <clocale> #include <cmath> #include <cstdio> +#include <random> #include <string> #include <vector> #include <ctime> @@ -156,7 +157,7 @@ static std::vector<std::string> split_string(const std::string& input, char deli int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); - srand(1234); + std::mt19937 rng(1234); common_params params; @@ -321,7 +322,7 @@ int main(int argc, char ** argv) { client.t_start_prompt = ggml_time_us(); client.t_start_gen = 0; - client.input = k_prompts[rand() % k_prompts.size()]; + client.input = k_prompts[rng() % k_prompts.size()]; client.response = ""; // construct the prompt: @@ -334,10 +335,10 @@ int main(int argc, char ** argv) { client.prompt += k_system; } - const int n_junk_cur = rand() % n_junk; + const int n_junk_cur = rng() % n_junk; for (int i = 0; i < n_junk_cur; ++i) { - const int r = rand() % k_questions.size(); + const int r = rng() % k_questions.size(); client.prompt += "User:\n" + k_questions[r] + "\nAssistant:\n " + k_answers[r] + "\n"; } client.prompt += "User:\n" + client.input + "\nAssistant:\n"; diff --git a/examples/pydantic_models_to_grammar.py b/examples/pydantic_models_to_grammar.py index 0cdd0b570935..736b2b7df104 100644 --- a/examples/pydantic_models_to_grammar.py +++ b/examples/pydantic_models_to_grammar.py @@ -1177,7 +1177,7 @@ def create_dynamic_model_from_function(func: Callable[..., Any]): dynamic_fields[param.name] = ( param.annotation if param.annotation != inspect.Parameter.empty else str, default_value) # Creating the dynamic model - dynamic_model = create_model(f"{getattr(func, '__name__')}", **dynamic_fields) + dynamic_model = create_model(f"{getattr(func, '__name__')}", **dynamic_fields) # ty: ignore[no-matching-overload] for name, param_doc in param_docs: dynamic_model.model_fields[name].description = param_doc.description diff --git a/examples/regex_to_grammar.py b/examples/regex_to_grammar.py deleted file mode 100644 index 5cd9210a4dfc..000000000000 --- a/examples/regex_to_grammar.py +++ /dev/null @@ -1,20 +0,0 @@ -import json, subprocess, sys, os - -assert len(sys.argv) >= 2 -[_, pattern, *rest] = sys.argv - -print(subprocess.check_output( - [ - "python", - os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "json_schema_to_grammar.py"), - *rest, - "-", - "--raw-pattern", - ], - text=True, - input=json.dumps({ - "type": "string", - "pattern": pattern, - }, indent=2))) diff --git a/examples/simple-cmake-pkg/CMakeLists.txt b/examples/simple-cmake-pkg/CMakeLists.txt index 128e38c8f2dc..04fe1291c23d 100644 --- a/examples/simple-cmake-pkg/CMakeLists.txt +++ b/examples/simple-cmake-pkg/CMakeLists.txt @@ -5,6 +5,9 @@ set(TARGET llama-simple-cmake-pkg) find_package(Llama REQUIRED) +# Check that repeated package discovery does not redefine imported targets. +find_package(Llama REQUIRED) + add_executable(${TARGET} ${CMAKE_CURRENT_LIST_DIR}/../simple/simple.cpp) install(TARGETS ${TARGET} RUNTIME) target_link_libraries(${TARGET} PRIVATE llama ggml::all ${CMAKE_THREAD_LIBS_INIT}) diff --git a/examples/speculative-simple/speculative-simple.cpp b/examples/speculative-simple/speculative-simple.cpp index 487ae03abfa7..863af5a2c71a 100644 --- a/examples/speculative-simple/speculative-simple.cpp +++ b/examples/speculative-simple/speculative-simple.cpp @@ -188,7 +188,7 @@ int main(int argc, char ** argv) { common_speculative_get_draft_params(spec, seq_id) = { /* .drafting = */ true, /* .n_max = */ n_draft_max, - /* .n_past = */ n_past, + /* .pos0 = */ n_past, /* .id_last = */ id_last, /* .prompt = */ &prompt_tgt, /* .result = */ &draft, // output diff --git a/examples/test-cmake/.gitignore b/examples/test-cmake/.gitignore index 0ddff317a4b5..b630ddb7d62f 100644 --- a/examples/test-cmake/.gitignore +++ b/examples/test-cmake/.gitignore @@ -1,3 +1,4 @@ llama-build-install install build +build-subdir diff --git a/examples/test-cmake/CMakeLists.txt b/examples/test-cmake/CMakeLists.txt index ed5cb1f3c262..6ceb3359e526 100644 --- a/examples/test-cmake/CMakeLists.txt +++ b/examples/test-cmake/CMakeLists.txt @@ -3,11 +3,20 @@ project(llama-simple) set(CMAKE_CXX_STANDARD 17) -find_package(llama 0.1.0 REQUIRED) +option(LLAMA_TEST_USE_SUBDIR "Use add_subdirectory instead of find_package" OFF) + +if(LLAMA_TEST_USE_SUBDIR) + add_subdirectory(../../ llama.cpp) +else() + find_package(llama 0.1.0 REQUIRED) +endif() add_executable(test-cmake test-cmake.cpp) target_link_libraries(test-cmake PRIVATE llama) -target_compile_definitions(test-cmake PRIVATE - LLAMA_BUILD_NUMBER=${LLAMA_BUILD_NUMBER} - LLAMA_BUILD_COMMIT="${LLAMA_BUILD_COMMIT}" -) + +if(DEFINED LLAMA_BUILD_NUMBER) + target_compile_definitions(test-cmake PRIVATE + LLAMA_BUILD_NUMBER=${LLAMA_BUILD_NUMBER} + LLAMA_BUILD_COMMIT="${LLAMA_BUILD_COMMIT}" + ) +endif() diff --git a/examples/test-cmake/README.md b/examples/test-cmake/README.md index 2f6a2fcfe9c3..03895abfb933 100644 --- a/examples/test-cmake/README.md +++ b/examples/test-cmake/README.md @@ -5,17 +5,18 @@ enable troubleshooting issues and exploration. The idea is that this can be used after making changes to llama.cpp installation cmake configuration and then verify it locally. -### Usage -The following will configure, build, and install llama.cpp +### find_package +The following will configure, build, and install llama.cpp, and the build a +project that uses find_package to use the installation. Configuring/build/install: ```console ./build-install.sh ``` The above command will create a directory named `install` in the current directory -which will have the follwing files in its lib directory: +which will have the following files in its lib directory: ```console -(venv) $ ls install/lib/ +$ ls install/lib/ cmake libggml.so libllama-common.so.0 libllama.so.0.1.0 llama.cpp libggml-base.so libggml.so.0 libllama-common.so.0.1.0 libmtmd.so pkgconfig libggml-base.so.0 libggml.so.0.19.0 libllama.so libmtmd.so.0 @@ -24,7 +25,7 @@ libggml-base.so.0.19.0 libllama-common.so libllama.so.0 libmtmd.so Build/run this project using the installation created above: ```console -(venv) $ ./build.sh +$ ./build.sh -- Configuring done (0.0s) -- Generating done (0.0s) -- Build files have been written to: /path/to/llama.cpp/examples/test-cmake/build @@ -34,3 +35,11 @@ Build/run this project using the installation created above: load_backend: loaded CPU backend from /path/to/llama.cpp/examples/test-cmake/install/lib/llama.cpp/libggml-cpu-alderlake.so [test-cmake] Backend initialized. ``` + +### add_subdirectory +The following will use add_subdirectory to include llama.cpp in a cmake project +and is intended to simulate projects that build llama.cpp in this way. + +```console +$ USE_SUBDIR=ON ./build.sh +``` diff --git a/examples/test-cmake/build.sh b/examples/test-cmake/build.sh index a212732b89d9..869a64160e6c 100755 --- a/examples/test-cmake/build.sh +++ b/examples/test-cmake/build.sh @@ -2,6 +2,17 @@ set -e -cmake -S . -B build -DCMAKE_PREFIX_PATH="${PWD}/install" -cmake --build build -LD_LIBRARY_PATH="${PWD}/install/lib/llama.cpp:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ./build/test-cmake +if [ "${USE_SUBDIR:-OFF}" = "ON" ]; then + BUILD_DIR="build-subdir" + CMAKE_ARGS="-DLLAMA_TEST_USE_SUBDIR=ON -DLLAMA_BUILD_COMMON=ON -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON-DLLAMA_BUILD_TESTS=ON" + LIB_PATH="${PWD}/${BUILD_DIR}/bin" +else + BUILD_DIR="build" + CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PWD}/install" + LIB_PATH="${PWD}/install/lib/llama.cpp" +fi + +cmake --fresh -S . -B "${BUILD_DIR}" ${CMAKE_ARGS} +cmake --build "${BUILD_DIR}" -j 8 + +LD_LIBRARY_PATH="${LIB_PATH}:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" "./${BUILD_DIR}/test-cmake" diff --git a/examples/test-cmake/test-cmake.cpp b/examples/test-cmake/test-cmake.cpp index c5c4765b439c..fea27c7e87b4 100644 --- a/examples/test-cmake/test-cmake.cpp +++ b/examples/test-cmake/test-cmake.cpp @@ -2,8 +2,13 @@ #include <cstdio> int main(void) { - printf("[test-cmake] version: %s, build: %d (%s)\n", +#ifdef LLAMA_BUILD_NUMBER + printf("[test-cmake] llama.cpp version: %s, build: %d (%s)\n", llama_version(), LLAMA_BUILD_NUMBER, LLAMA_BUILD_COMMIT); +#else + printf("[test-cmake] llama.cpp version: %s\n", llama_version()); +#endif + printf("[test-cmake] ggml version: %s, commit: %s\n", ggml_version(), ggml_commit()); printf("[test-cmake] Initializing backend...\n"); llama_backend_init(); printf("[test-cmake] Backend initialized.\n"); diff --git a/examples/ts-type-to-grammar.sh b/examples/ts-type-to-grammar.sh deleted file mode 100755 index 966050407888..000000000000 --- a/examples/ts-type-to-grammar.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# ./examples/ts-type-to-grammar.sh "{a:string,b:string,c?:string}" -# python examples/json_schema_to_grammar.py https://json.schemastore.org/tsconfig.json -# -set -euo pipefail - -readonly type="$1" - -# Create a temporary directory -TMPDIR="" -trap 'rm -fR "$TMPDIR"' EXIT -TMPDIR=$(mktemp -d) - -DTS_FILE="$TMPDIR/type.d.ts" -SCHEMA_FILE="$TMPDIR/schema.json" - -echo "export type MyType = $type" > "$DTS_FILE" - -# This is a fork of typescript-json-schema, actively maintained as of March 2024: -# https://github.com/vega/ts-json-schema-generator -npx ts-json-schema-generator --unstable --no-top-ref --path "$DTS_FILE" --type MyType -e none > "$SCHEMA_FILE" - -# Alternative, not actively maintained as of March 2024: -# https://github.com/YousefED/typescript-json-schema -# npx typescript-json-schema --defaultProps --required "$DTS_FILE" MyType | tee "$SCHEMA_FILE" >&2 - -./examples/json_schema_to_grammar.py "$SCHEMA_FILE" diff --git a/flake.nix b/flake.nix index bb02c8e52f9a..6373d3b0be3d 100644 --- a/flake.nix +++ b/flake.nix @@ -128,7 +128,7 @@ }: { # For standardised reproducible formatting with `nix fmt` - formatter = pkgs.nixfmt-rfc-style; + formatter = pkgs.nixfmt; # Unlike `.#packages`, legacyPackages may contain values of # arbitrary types (including nested attrsets) and may even throw @@ -156,7 +156,7 @@ windows = config.legacyPackages.llamaPackagesWindows.llama-cpp; python-scripts = config.legacyPackages.llamaPackages.python-scripts; } - // lib.optionalAttrs pkgs.stdenv.isLinux { + // lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { cuda = config.legacyPackages.llamaPackagesCuda.llama-cpp; mpi-cpu = config.packages.default.override { useMpi = true; }; diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index d76ed8ab0497..1df78fddfd5d 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -4,8 +4,8 @@ project("ggml" C CXX ASM) ### GGML Version set(GGML_VERSION_MAJOR 0) -set(GGML_VERSION_MINOR 23) -set(GGML_VERSION_PATCH 0) +set(GGML_VERSION_MINOR 25) +set(GGML_VERSION_PATCH 1) set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") @@ -204,6 +204,8 @@ option(GGML_CUDA_NO_PEER_COPY "ggml: do not use peer to peer copie option(GGML_CUDA_NO_VMM "ggml: do not try to use CUDA VMM" OFF) option(GGML_CUDA_FA "ggml: compile ggml FlashAttention CUDA kernels" ON) option(GGML_CUDA_FA_ALL_QUANTS "ggml: compile all quants for FlashAttention" OFF) +set (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16" CACHE STRING + "ggml: FlashAttention K-V type combinations to compile, \"all\" or a list such as \"q8_0-q8_0;q8_0-q4_0\"") option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT}) option(GGML_CUDA_NCCL "ggml: use NVIDIA Collective Comm. Library" ON) set (GGML_CUDA_COMPRESSION_MODE "size" CACHE STRING diff --git a/ggml/cmake/common.cmake b/ggml/cmake/common.cmake index cb6638833204..5d4c4bfdd71f 100644 --- a/ggml/cmake/common.cmake +++ b/ggml/cmake/common.cmake @@ -48,3 +48,89 @@ function(ggml_get_system_arch) set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE) endif() endfunction() + +# Determines which FlashAttention vector kernel template instances to compile, returns them in OUT_SRCS. +function(ggml_cuda_fattn_vec_instances DIR OUT_SRCS) + set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16) + + string(TOLOWER "${GGML_CUDA_FA_QUANTS}" FA_QUANTS) + string(STRIP "${FA_QUANTS}" FA_QUANTS) + if (GGML_CUDA_FA_ALL_QUANTS) + message(WARNING "GGML_CUDA_FA_ALL_QUANTS is deprecated, use GGML_CUDA_FA_QUANTS=all instead") + set(FA_QUANTS all) + endif() + if (NOT FA_QUANTS) + message(FATAL_ERROR "GGML_CUDA_FA_QUANTS must not be empty") + endif() + + if (FA_QUANTS STREQUAL "all") + set(FA_COMBINATIONS "") + foreach (TYPE_V IN LISTS FA_TYPES) + foreach (TYPE_K IN LISTS FA_TYPES) + list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V}) + endforeach() + endforeach() + else() + set(FA_COMBINATIONS f16-f16) + + string(REPLACE "," ";" FA_SELECTED "${FA_QUANTS}") + foreach (COMBINATION IN LISTS FA_SELECTED) + string(STRIP "${COMBINATION}" COMBINATION) + if (NOT COMBINATION MATCHES "^([a-z0-9_]+)-([a-z0-9_]+)$") + message(FATAL_ERROR "GGML_CUDA_FA_QUANTS: \"${COMBINATION}\" is not \"all\" or a <type_K>-<type_V> combination") + endif() + set(TYPE_K ${CMAKE_MATCH_1}) + set(TYPE_V ${CMAKE_MATCH_2}) + foreach (TYPE ${TYPE_K} ${TYPE_V}) + if (NOT TYPE IN_LIST FA_TYPES) + message(FATAL_ERROR + "GGML_CUDA_FA_QUANTS: unknown type \"${TYPE}\" in \"${COMBINATION}\", must be one of: ${FA_TYPES}") + endif() + endforeach() + list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V}) + endforeach() + endif() + # fork: TurboQuant KV cache combinations are always compiled, whatever GGML_CUDA_FA_QUANTS says. + set(FA_TURBO_COMBINATIONS + turbo3_0-turbo3_0 turbo3_0-q8_0 q8_0-turbo3_0 + turbo2_0-turbo2_0 turbo2_0-q8_0 q8_0-turbo2_0 + turbo3_0-turbo2_0 turbo2_0-turbo3_0 + turbo4_0-turbo4_0 turbo4_0-q8_0 q8_0-turbo4_0 + turbo4_0-turbo3_0 turbo3_0-turbo4_0 + turbo4_0-turbo2_0 turbo2_0-turbo4_0) + foreach (COMBINATION IN LISTS FA_TURBO_COMBINATIONS) + string(REPLACE "-" "_" COMBINATION_DEF "GGML_CUDA_FA_${COMBINATION}") + string(TOUPPER "${COMBINATION_DEF}" COMBINATION_DEF) + add_compile_definitions(${COMBINATION_DEF}=1) + endforeach() + list(APPEND FA_COMBINATIONS ${FA_TURBO_COMBINATIONS}) + + list(REMOVE_DUPLICATES FA_COMBINATIONS) + + string(REPLACE ";" "," FA_QUANTS_DEFINE "${FA_QUANTS}") + add_compile_definitions(GGML_CUDA_FA_QUANTS="${FA_QUANTS_DEFINE}") + foreach (TYPE_V IN LISTS FA_TYPES) + foreach (TYPE_K IN LISTS FA_TYPES) + if ("${TYPE_K}-${TYPE_V}" IN_LIST FA_COMBINATIONS) + set(COMPILED 1) + else() + set(COMPILED 0) + endif() + string(TOUPPER "GGML_CUDA_FA_${TYPE_K}_${TYPE_V}" COMBINATION_DEF) + add_compile_definitions(${COMBINATION_DEF}=${COMPILED}) + endforeach() + endforeach() + + message(STATUS "FlashAttention K-V type combinations: ${FA_COMBINATIONS}") + + set(SRCS "") + foreach (COMBINATION IN LISTS FA_COMBINATIONS) + set(SRC "${DIR}/template-instances/fattn-vec-instance-${COMBINATION}.cu") + if (NOT EXISTS "${SRC}") + message(FATAL_ERROR "FlashAttention template instance \"${SRC}\" does not exist") + endif() + list(APPEND SRCS "${SRC}") + endforeach() + + set(${OUT_SRCS} ${SRCS} PARENT_SCOPE) +endfunction() diff --git a/ggml/include/ggml-rpc.h b/ggml/include/ggml-rpc.h index e1b140cec0e1..66ca14b4115c 100644 --- a/ggml/include/ggml-rpc.h +++ b/ggml/include/ggml-rpc.h @@ -6,7 +6,7 @@ extern "C" { #endif -#define RPC_PROTO_MAJOR_VERSION 6 +#define RPC_PROTO_MAJOR_VERSION 7 #define RPC_PROTO_MINOR_VERSION 0 // fork patch version 4: 109 + FP8_B128 phase 2's 2 new ops (FP8_QUANT_ROT, FP8_MUL_MAT) = 111 // fork patch version 5: 111 + WP_DSV41_SPARSE_ATTN's GGML_OP_SPARSE_ATTN_DSV4 = 112 diff --git a/ggml/include/ggml-sycl.h b/ggml/include/ggml-sycl.h index 418a7ba978b4..1e353ffa31c7 100644 --- a/ggml/include/ggml-sycl.h +++ b/ggml/include/ggml-sycl.h @@ -25,7 +25,7 @@ GGML_BACKEND_API bool ggml_backend_is_sycl(ggml_backend_t backend); GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device); // split tensor buffer that splits matrices by rows across multiple devices -GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split); +GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(int main_device, const float * tensor_split); // Tensor parallelism (--split-mode tensor): comm_init/free/allreduce_tensor // trio queried by the meta-backend via ggml_backend_reg_get_proc_address. @@ -36,6 +36,8 @@ GGML_BACKEND_API void ggml_backend_sycl_comm_free(void * comm_ctx); GGML_BACKEND_API bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx, struct ggml_tensor ** tensors); // pinned host buffer for use with the CPU backend for faster copies between CPU and GPU +// pins on device 0 - a copy between another device and this memory can fail, +// use ggml_backend_dev_host_buffer_type to pin on the device that does the copy GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type(void); GGML_BACKEND_API void ggml_backend_sycl_print_sycl_devices(void); @@ -43,7 +45,7 @@ GGML_BACKEND_API void ggml_backend_sycl_get_gpu_list(int *id_list, int max_len); GGML_BACKEND_API void ggml_backend_sycl_get_device_description(int device, char *description, size_t description_size); -GGML_BACKEND_API int ggml_backend_sycl_get_device_count(); +GGML_BACKEND_API int ggml_backend_sycl_get_device_count(void); GGML_BACKEND_API void ggml_backend_sycl_get_device_memory(int device, size_t *free, size_t *total); // SYCL doesn't support registering host memory, keep here for reference diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 9e066b5aae46..426da308cee8 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -453,10 +453,21 @@ extern "C" { GGML_TYPE_COUNT = 59, }; - // precision + // [TAG_GGML_PREC] + // this enum is used to declare the allowed numerical precision/data-types types that can be used during the compute of an op + // the declared types can be: + // - result accumulation type + // - source tensor data representation type + // - etc. + // the precision parameters are stored as ggml_tensor.op_params to the respective ops enum ggml_prec { - GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default - GGML_PREC_F32 = 10, + GGML_PREC_UNDEFINED = 0, + GGML_PREC_DEFAULT = 0, // note: deprecated, use GGML_PREC_UNDEFINED + GGML_PREC_F32 = 10, + GGML_PREC_BF16 = 15, + GGML_PREC_F16 = 20, + GGML_PREC_Q8 = 30, + GGML_PREC_Q4 = 40, }; // op hint @@ -1588,6 +1599,42 @@ extern "C" { struct ggml_tensor * b, float eps); + // [TAG_GGML_PREC] + // set the minimum required accumulator type for the implementation to use during the compute + // for example: + // - GGML_PREC_F32 - requires accumulation of the results in F32 + // - GGML_PREC_BF16 - can accumulate the results in BF16, F32 + // - GGML_PREC_F16 - can accumulate the results in F16, F32 + // - GGML_PREC_Q8 - not allowed + // - GGML_PREC_Q4 - not allowed + // + // return false on faliure + GGML_API bool ggml_prec_set_acc( + struct ggml_tensor * a, + enum ggml_prec prec); + + // [TAG_GGML_PREC] + // set the smallest rank that the implementation can use to internally convert the src[idx] data to + // ranks in decreasing order: + // - GGML_PREC_F32 - GGML_TYPE_F32 + // - GGML_PREC_BF16 - GGML_TYPE_BF16 + // - GGML_PREC_F16 - GGML_TYPE_F16, + // - GGML_PREC_Q8 - GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, GGML_TYPE_Q8_K, etc. + // - GGML_PREC_Q4 - GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_NVFP4, GGML_TYPE_MXFP4, etc. + // + // for example: + // - ggml_prec_set_src(a, GGML_PREC_Q8, 1): + // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to GGML_TYPE_Q8_0 + // - cannot quantize it down to GGML_TYPE_Q4_0 or GGML_TYPE_NVFP4 + // - ggml_prec_set_src(a, GGML_PREC_Q4, 1): + // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to 4-bit datatypes such as GGML_TYPE_Q4_K, GGML_TYPE_NVFP4 etc. + // + // return false on faliure + GGML_API bool ggml_prec_set_src( + struct ggml_tensor * a, + enum ggml_prec prec, + int idx); + // A: k columns, n rows => [ne03, ne02, n, k] // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k] // result is n columns, m rows => [ne03 * x, ne02 * y, m, n] @@ -1598,9 +1645,10 @@ extern "C" { // change the precision of a matrix multiplication // set to GGML_PREC_F32 for higher precision (useful for phi-2) - GGML_API void ggml_mul_mat_set_prec( + GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec( struct ggml_tensor * a, - enum ggml_prec prec); + enum ggml_prec prec), + "use ggml_prec_set_acc() instead"); // change the hint of a matrix multiplication GGML_API void ggml_mul_mat_set_hint( @@ -2621,9 +2669,10 @@ extern "C" { float max_bias, float logit_softcap); - GGML_API void ggml_flash_attn_ext_set_prec( + GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec( struct ggml_tensor * a, - enum ggml_prec prec); + enum ggml_prec prec), + "use ggml_prec_set_acc() instead"); GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( const struct ggml_tensor * a); @@ -2898,11 +2947,21 @@ extern "C" { struct ggml_tensor * x, struct ggml_tensor * weights); + // hc_pre with a per-element gate (Qwen3.8-Flash-Next): gate [n_embd, hc, n_tokens] + // result[i, t] = scale*sum_h x[i, h, t]*sigmoid(gate[i, h, t]) + // + GGML_API struct ggml_tensor * ggml_dsv4_hc_pre_gated( + struct ggml_context * ctx, + struct ggml_tensor * x, + struct ggml_tensor * gate, + float scale); + // hc_post: x [n_embd, n_tokens], residual [n_embd, hc, n_tokens], // post [hc, n_tokens], comb [dst_hc, src_hc, n_tokens] // -> [n_embd, hc, n_tokens] // result[i, dst, t] = x[i, t]*post[dst, t] // + sum_src residual[i, src, t]*comb[dst, src, t] + // comb == NULL uses the identity: result[i, dst, t] = x[i, t]*post[dst, t] + residual[i, dst, t] // GGML_API struct ggml_tensor * ggml_dsv4_hc_post( struct ggml_context * ctx, diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 12e52d79e29b..5ad2747a36cb 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -2092,11 +2092,6 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m ggml_context * simple_ctx = stc.ctxs[j].get(); ggml_backend_buffer_t simple_buf = buf_ctx->bufs[j].get(); - if ((simple_buf != nullptr) && ggml_backend_buffer_is_multi_buffer(simple_buf)) { - // see https://github.com/ggml-org/llama.cpp/issues/22197 - GGML_ABORT("multi buffers are not supported by the meta backend"); - } - if (split_dim >= 0 && split_dim < GGML_MAX_DIMS) { // TODO: the following assert fails for llama-parallel even though the results are correct: // GGML_ASSERT(ggml_is_contiguously_allocated(tensor)); @@ -2144,16 +2139,27 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m } } } + // TODO: revisit once the graph allocator has been refactored, see https://github.com/ggml-org/llama.cpp/pull/25051#issuecomment-4842873396 + ggml_backend_buffer_t init_buf = simple_buf; if (t_ij->view_src != nullptr) { t_ij->data = (char *) t_ij->view_src->data + t_ij->view_offs; + // views inherit the source slice's concrete sub-buffer (issue 22197) + if (tensor->view_src != nullptr && ggml_backend_buffer_is_meta(tensor->view_src->buffer) + && t_ij->view_src->buffer != nullptr) { + t_ij->buffer = t_ij->view_src->buffer; + init_buf = t_ij->view_src->buffer; + } } else if (simple_buf != nullptr) { + if (ggml_backend_buffer_is_multi_buffer(simple_buf)) { + GGML_ABORT("multi buffers are not supported by the meta backend"); + } t_ij->data = (char *) ggml_backend_buffer_get_base(simple_buf) + size_t(tensor->data) - size_t(ggml_backend_buffer_get_base(tensor->buffer)); } - if (simple_buf) { + if (init_buf) { // the backend that owns the buffer will set .extra - ggml_backend_buffer_init_tensor(simple_buf, t_ij); + ggml_backend_buffer_init_tensor(init_buf, t_ij); } else { t_ij->extra = tensor->extra; } diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 18a556496d6a..565fd603cf65 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1156,7 +1156,7 @@ static void ggml_backend_sched_split_inputs_grow(struct ggml_backend_sched_split int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS; if (split->inputs_capacity > 0) { new_cap = 2*split->inputs_capacity; - GGML_LOG_WARN("%s: increasing split inputs capacity from %d to %d\n", __func__, split->inputs_capacity, new_cap); + GGML_LOG_DEBUG("%s: increasing split inputs capacity from %d to %d\n", __func__, split->inputs_capacity, new_cap); } auto * pnew = (struct ggml_tensor **) realloc((void *) split->inputs, new_cap * sizeof(struct ggml_tensor *)); if (pnew == NULL) { @@ -1171,7 +1171,7 @@ static void ggml_backend_sched_graph_inputs_grow(ggml_backend_sched_t sched) { int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS; if (sched->graph_inputs_capacity > 0) { new_cap = 2*sched->graph_inputs_capacity; - GGML_LOG_WARN("%s: increasing graph inputs capacity from %d to %d\n", __func__, sched->graph_inputs_capacity, new_cap); + GGML_LOG_DEBUG("%s: increasing graph inputs capacity from %d to %d\n", __func__, sched->graph_inputs_capacity, new_cap); } auto * pnew = (struct ggml_tensor **) realloc((void *) sched->graph_inputs, new_cap * sizeof(struct ggml_tensor *)); if (pnew == NULL) { @@ -1646,17 +1646,6 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra break; } } - // check if the split has too many inputs - // FIXME: count the number of inputs instead of only checking when full - if (split->n_inputs >= split->inputs_capacity) { - const size_t id = hash_id(src); - int src_backend_id = sched->hv_tensor_backend_ids[id]; - bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id); - if (src_backend_id != cur_backend_id && tensor_id_copy(id, cur_backend_id, 0) == NULL && !supported) { - need_new_split = true; - break; - } - } } } @@ -2039,7 +2028,10 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { ggml_backend_synchronize(sched->backends[i]); } - ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids); + if (!ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids)) { + GGML_LOG_ERROR("%s: failed to reserve graph buffers\n", __func__); + return false; + } if (!ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) { GGML_LOG_ERROR("%s: failed to allocate graph\n", __func__); return false; @@ -2278,6 +2270,10 @@ struct ggml_backend_sched_compute_runner { ggml_tensor * ids_tensor = node->src[2]; ggml_backend_t ids_backend = split_backend; + if (ggml_nelements(ids_tensor) == 0) { + continue; + } + // if the ids tensor is also an input of the split, it may not have been copied yet to the split backend // in that case, we use the original ids tensor for (int i = input_id + 1; i < split->n_inputs; i++) { diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 6560bbcdc166..cf9a9178706f 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -522,7 +522,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name) elseif (GGML_SYSTEM_ARCH STREQUAL "s390x") message(STATUS "s390x detected") list(APPEND GGML_CPU_SOURCES - ggml-cpu/arch/s390/quants.c) + ggml-cpu/arch/s390/quants.c + ggml-cpu/arch/s390/repack.cpp + ) # for native compilation if (GGML_NATIVE) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 152e0bac99b0..4dbd1982ba03 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -39,6 +39,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -55,6 +57,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -87,6 +91,8 @@ // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K @@ -98,6 +104,8 @@ #define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K @@ -124,6 +132,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -140,6 +150,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -171,6 +183,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -187,6 +201,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -213,6 +229,8 @@ #define ggml_quantize_mat_q8_K_4x1_generic ggml_quantize_mat_q8_K_4x1 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K @@ -228,6 +246,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K @@ -247,7 +267,6 @@ // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 -#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K @@ -260,11 +279,11 @@ #define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K #define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K // repack.cpp -#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 -#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K @@ -280,7 +299,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 -#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K @@ -318,6 +338,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -334,6 +356,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index a7534443091f..ad0e5ccaf7b7 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -48,6 +48,24 @@ static inline void decode_q_Kx8_6bit_scales(const uint8_t * scales_in, int16x8_t } #endif +#if defined(__aarch64__) && defined(__ARM_NEON) && (defined(__ARM_FEATURE_DOTPROD) || defined(__ARM_FEATURE_MATMUL_INT8)) +#define B1(c,s,n) 0x ## n ## c , 0x ## n ## s +#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s) +#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s) +#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s) +#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s) +#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s) +#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s) +#define B8(c,s ) B7(c,s, c), B7(c,s, s) + +static const uint64_t table_q1_signs[256] = { B8(ff, 01) }; + +static inline int8x16_t ggml_q1_0_unpack_pair(uint8_t bits0, uint8_t bits1) { + return vreinterpretq_s8_u8(vcombine_u8(vcreate_u8(table_q1_signs[bits0]), + vcreate_u8(table_q1_signs[bits1]))); +} +#endif + void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { assert(QK8_0 == 32); assert(k % QK8_0 == 0); @@ -1823,6 +1841,132 @@ void ggml_gemv_q8_0_4x8_q8_0(int n, ggml_gemv_q8_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); } +void ggml_gemv_q1_0_4x4_q8_0(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(nb); + UNUSED(ncols_interleaved); + +#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) + for (int c = 0; c < nc; c += ncols_interleaved) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (c / ncols_interleaved) * nb; + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + float32x4_t acc = vdupq_n_f32(0); + + for (int l = 0; l < nb; l++) { + const float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d)); + float32x4_t accb = vdupq_n_f32(0); + + for (int k = 0; k < 4; k++) { + const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * 4 + k; + const float ad = GGML_CPU_FP16_TO_FP32(a_blk->d); + int32x4_t ret = vdupq_n_s32(0); + + for (int tile = 0; tile < 8; tile += 4) { + const int8x16_t signs0 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 0) + 0], + b_ptr[l].qs[k * 16 + 2 * (tile + 0) + 1]); + const int8x16_t signs1 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 1) + 0], + b_ptr[l].qs[k * 16 + 2 * (tile + 1) + 1]); + const int8x16_t signs2 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 2) + 0], + b_ptr[l].qs[k * 16 + 2 * (tile + 2) + 1]); + const int8x16_t signs3 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 3) + 0], + b_ptr[l].qs[k * 16 + 2 * (tile + 3) + 1]); + const int8x16_t q_tiles = vld1q_s8(a_blk->qs + tile * 4); + + ret = vdotq_laneq_s32(ret, signs0, q_tiles, 0); + ret = vdotq_laneq_s32(ret, signs1, q_tiles, 1); + ret = vdotq_laneq_s32(ret, signs2, q_tiles, 2); + ret = vdotq_laneq_s32(ret, signs3, q_tiles, 3); + } + + accb = vfmaq_n_f32(accb, vcvtq_f32_s32(ret), ad); + } + acc = vfmaq_f32(acc, accb, b_d); + } + vst1q_f32(s, acc); + s += ncols_interleaved; + } + return; +#endif + ggml_gemv_q1_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_q1_0_4x8_q8_0(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(nb); + UNUSED(ncols_interleaved); + +#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) + for (int c = 0; c < nc; c += ncols_interleaved) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (c / ncols_interleaved) * nb; + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + float32x4_t acc = vdupq_n_f32(0); + + for (int l = 0; l < nb; l++) { + const float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d)); + float32x4_t accb = vdupq_n_f32(0); + + for (int k = 0; k < 4; ++k) { + const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * 4 + k; + const uint8_t * GGML_RESTRICT b_qs = (const uint8_t *) b_ptr[l].qs + k * 16; + const float ad = GGML_CPU_FP16_TO_FP32(a_blk->d); + + int8x8x4_t a_chunks = vld1_s8_x4(a_blk->qs); + int8x16_t a0 = vcombine_s8(a_chunks.val[0], a_chunks.val[0]); + int8x16_t a1 = vcombine_s8(a_chunks.val[1], a_chunks.val[1]); + int8x16_t a2 = vcombine_s8(a_chunks.val[2], a_chunks.val[2]); + int8x16_t a3 = vcombine_s8(a_chunks.val[3], a_chunks.val[3]); + + int32x4_t ret0 = vdupq_n_s32(0); + int32x4_t ret1 = vdupq_n_s32(0); + + ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[0], b_qs[1]), a0); + ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[2], b_qs[3]), a0); + ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[4], b_qs[5]), a1); + ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[6], b_qs[7]), a1); + ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[8], b_qs[9]), a2); + ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[10], b_qs[11]), a2); + ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[12], b_qs[13]), a3); + ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[14], b_qs[15]), a3); + + accb = vfmaq_n_f32(accb, vcvtq_f32_s32(vpaddq_s32(ret0, ret1)), ad); + } + + acc = vfmaq_f32(acc, accb, b_d); + } + + vst1q_f32(s, acc); + s += ncols_interleaved; + } + return; +#endif + + ggml_gemv_q1_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} + void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK8_0; const int nb = n / qk; @@ -5154,3 +5298,168 @@ void ggml_gemm_q8_0_4x8_q8_0(int n, #endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) ggml_gemm_q8_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); } + +void ggml_gemm_q1_0_4x4_q8_0(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(n % qk == 0); + assert(nr % 4 == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(nb); + UNUSED(ncols_interleaved); + +#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + float32x4_t sumf[4]; + for (int m = 0; m < 4; m++) { + sumf[m] = vdupq_n_f32(0); + } + + for (int l = 0; l < nb; l++) { + float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d)); + float32x4_t blockf_0 = vdupq_n_f32(0); + float32x4_t blockf_1 = vdupq_n_f32(0); + float32x4_t blockf_2 = vdupq_n_f32(0); + float32x4_t blockf_3 = vdupq_n_f32(0); + + for (int k = 0; k < 4; ++k) { + const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k; + float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *) a_blk->d)); + + int32x4_t sumi_0 = vdupq_n_s32(0); + int32x4_t sumi_1 = vdupq_n_s32(0); + int32x4_t sumi_2 = vdupq_n_s32(0); + int32x4_t sumi_3 = vdupq_n_s32(0); + + for (int tile = 0; tile < 8; ++tile) { + const int8x16_t signs = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * tile + 0], + b_ptr[l].qs[k * 16 + 2 * tile + 1]); + const int8x16_t a_tile = vld1q_s8(a_blk->qs + tile * 16); + + sumi_0 = vdotq_laneq_s32(sumi_0, signs, a_tile, 0); + sumi_1 = vdotq_laneq_s32(sumi_1, signs, a_tile, 1); + sumi_2 = vdotq_laneq_s32(sumi_2, signs, a_tile, 2); + sumi_3 = vdotq_laneq_s32(sumi_3, signs, a_tile, 3); + } + + blockf_0 = vfmaq_laneq_f32(blockf_0, vcvtq_f32_s32(sumi_0), a_d, 0); + blockf_1 = vfmaq_laneq_f32(blockf_1, vcvtq_f32_s32(sumi_1), a_d, 1); + blockf_2 = vfmaq_laneq_f32(blockf_2, vcvtq_f32_s32(sumi_2), a_d, 2); + blockf_3 = vfmaq_laneq_f32(blockf_3, vcvtq_f32_s32(sumi_3), a_d, 3); + } + + sumf[0] = vfmaq_f32(sumf[0], blockf_0, b_d); + sumf[1] = vfmaq_f32(sumf[1], blockf_1, b_d); + sumf[2] = vfmaq_f32(sumf[2], blockf_2, b_d); + sumf[3] = vfmaq_f32(sumf[3], blockf_3, b_d); + } + + for (int m = 0; m < 4; m++) { + vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]); + } + } + } + return; +#endif + ggml_gemm_q1_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_q1_0_4x8_q8_0(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(n % qk == 0); + assert(nr % 4 == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(nb); + UNUSED(ncols_interleaved); + +#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb); + + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + float32x4_t sumf[4]; + for (int m = 0; m < 4; ++m) { + sumf[m] = vdupq_n_f32(0); + } + + for (int l = 0; l < nb; l++) { + const float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d)); + float32x4_t blockf[4]; + for (int m = 0; m < 4; ++m) { + blockf[m] = vdupq_n_f32(0); + } + + for (int k = 0; k < 4; ++k) { + const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k; + const uint8_t * GGML_RESTRICT b_qs = (const uint8_t *) b_ptr[l].qs + k * 16; + + int32x4_t acc[4]; + for (int i = 0; i < 4; ++i) { + acc[i] = vdupq_n_s32(0); + } + + for (int chunk = 0; chunk < 4; ++chunk) { + const int8x16_t a01 = vld1q_s8(a_blk->qs + chunk * 32); + const int8x16_t a23 = vld1q_s8(a_blk->qs + chunk * 32 + 16); + const int8x16_t b01 = ggml_q1_0_unpack_pair(b_qs[chunk * 4 + 0], b_qs[chunk * 4 + 1]); + const int8x16_t b23 = ggml_q1_0_unpack_pair(b_qs[chunk * 4 + 2], b_qs[chunk * 4 + 3]); + + acc[0] = vmmlaq_s32(acc[0], a01, b01); + acc[1] = vmmlaq_s32(acc[1], a01, b23); + acc[2] = vmmlaq_s32(acc[2], a23, b01); + acc[3] = vmmlaq_s32(acc[3], a23, b23); + } + + const int32x4_t row0 = vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])); + const int32x4_t row1 = vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1])); + const int32x4_t row2 = vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])); + const int32x4_t row3 = vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3])); + const float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *) a_blk->d)); + + blockf[0] = vfmaq_laneq_f32(blockf[0], vcvtq_f32_s32(row0), a_d, 0); + blockf[1] = vfmaq_laneq_f32(blockf[1], vcvtq_f32_s32(row1), a_d, 1); + blockf[2] = vfmaq_laneq_f32(blockf[2], vcvtq_f32_s32(row2), a_d, 2); + blockf[3] = vfmaq_laneq_f32(blockf[3], vcvtq_f32_s32(row3), a_d, 3); + } + + sumf[0] = vfmaq_f32(sumf[0], blockf[0], b_d); + sumf[1] = vfmaq_f32(sumf[1], blockf[1], b_d); + sumf[2] = vfmaq_f32(sumf[2], blockf[2], b_d); + sumf[3] = vfmaq_f32(sumf[3], blockf[3], b_d); + } + + for (int m = 0; m < 4; ++m) { + vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]); + } + } + } + return; +#endif + + ggml_gemm_q1_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} diff --git a/ggml/src/ggml-cpu/arch/s390/quants.c b/ggml/src/ggml-cpu/arch/s390/quants.c index d3436c24b5f3..52344828e39d 100644 --- a/ggml/src/ggml-cpu/arch/s390/quants.c +++ b/ggml/src/ggml-cpu/arch/s390/quants.c @@ -146,6 +146,74 @@ void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, i //===================================== Dot products ================================= +void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK1_0; // 128 + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + +#if defined(__VXE__) || defined(__VXE2__) + float32x4_t v_sumf = vec_splats(0.0f); + + const uint8x16_t v_zero = vec_splats((uint8_t)0x00); // zero + const uint8x16_t v_bias = vec_splats((uint8_t)0x80); // bias from signed to unsigned + // v ^ 0x80 == v + 128 + + const uint8x16_t v_idx = (const uint8x16_t){ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 }; + const uint8x16_t v_bit = (const uint8x16_t){ 1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128 }; + + for (int i = 0; i < nb; ++i) { + const uint8x16_t v_x = vec_xl(0, (const uint8_t *)x[i].qs); + const float32x4_t v_xd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d)); + + for (int k = 0; k < 4; ++k) { + // sub-block k holds elements 32k .. 32k+31 + const block_q8_0 * GGML_RESTRICT yb = &y[i*4 + k]; + const float32x4_t v_yd = vec_splats(GGML_CPU_FP16_TO_FP32(yb->d)); + + const uint8x16_t v_xrl = vec_perm(v_x, v_x, vec_add(v_idx, vec_splats((uint8_t)(k*4 + 0)))); + const uint8x16_t v_xrh = vec_perm(v_x, v_x, vec_add(v_idx, vec_splats((uint8_t)(k*4 + 2)))); + + // isolate each lane's bit, then set all ones where that bit is clear, the -d case + const int8x16_t v_ml = (int8x16_t)vec_cmpeq(vec_and(v_xrl, v_bit), v_zero); + const int8x16_t v_mh = (int8x16_t)vec_cmpeq(vec_and(v_xrh, v_bit), v_zero); + + const int8x16_t v_yl = vec_xl(0, (const int8_t *)yb->qs); + const int8x16_t v_yh = vec_xl(QK8_0/2, (const int8_t *)yb->qs); + + // weights are only +1 or -1, so negate y + const int8x16_t v_ysl = vec_sub(vec_xor(v_yl, v_ml), v_ml); + const int8x16_t v_ysh = vec_sub(vec_xor(v_yh, v_mh), v_mh); + + // bias to unsigned, then vec_sum4 adds each group of 4 bytes into one word + const uint32x4_t v_p = vec_add(vec_sum4(vec_xor((uint8x16_t)v_ysl, v_bias), v_zero), + vec_sum4(vec_xor((uint8x16_t)v_ysh, v_bias), v_zero)); + + // each word summed 8 biased bytes, so take back 8 * 128 + const int32x4_t v_xy = vec_sub((int32x4_t)v_p, vec_splats((int32_t)1024)); + + // apply both block scales and add into the running total + v_sumf = vec_madd(vec_float(v_xy), vec_mul(v_xd, v_yd), v_sumf); + } + } + + *s = vec_hsum_f32x4(v_sumf); +#else + UNUSED(nb); + UNUSED(x); + UNUSED(y); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; @@ -349,6 +417,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo sumf = vec_hsum_f32x4(v_acc); *s = sumf; #else + UNUSED(nb); UNUSED(x); UNUSED(y); UNUSED(ib); diff --git a/ggml/src/ggml-cpu/arch/s390/repack.cpp b/ggml/src/ggml-cpu/arch/s390/repack.cpp new file mode 100644 index 000000000000..abf3433adda2 --- /dev/null +++ b/ggml/src/ggml-cpu/arch/s390/repack.cpp @@ -0,0 +1,225 @@ +#define GGML_COMMON_IMPL_CPP +#define GGML_COMMON_DECL_CPP +#include "ggml-common.h" +#include "ggml-backend-impl.h" + +#include "ggml-impl.h" +#include "ggml-cpu.h" +#include "ggml-cpu-impl.h" +#include "simd-mappings.h" +#include "traits.h" + +#include <cmath> +#include <cstring> +#include <cassert> + +#define GGML_CPU_CLANG_WORKAROUND +#include "../../repack.h" + +#define UNUSED GGML_UNUSED + +void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + assert(QK8_0 == 32); + assert(k % QK8_0 == 0); + const int nb = k / QK8_0; + + block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy; + +#if defined(__VXE__) || defined(__VXE2__) + float32x4_t v_src[4][8]; + float id[4]; + + for (int i = 0; i < nb; i++) { + float32x4_t v_asrc[8]; + float32x4_t v_amax[8]; + + for (int row_iter = 0; row_iter < 4; row_iter++) { + for (int j = 0; j < 8; j++) v_src[row_iter][j] = vec_xl(0, x + row_iter * k + i * 32 + 4 * j); + for (int j = 0; j < 8; j++) v_asrc[j] = vec_abs(v_src[row_iter][j]); + + for (int j = 0; j < 4; j++) v_amax[2 * j] = vec_max(v_asrc[2 * j], v_asrc[2 * j + 1]); + for (int j = 0; j < 2; j++) v_amax[4 * j] = vec_max(v_amax[4 * j], v_amax[4 * j + 2]); + for (int j = 0; j < 1; j++) v_amax[8 * j] = vec_max(v_amax[8 * j], v_amax[8 * j + 4]); + + const float amax = MAX(MAX(vec_extract(v_amax[0], 0), vec_extract(v_amax[0], 1)), + MAX(vec_extract(v_amax[0], 2), vec_extract(v_amax[0], 3))); + + const float d = amax / ((1 << 7) - 1); + id[row_iter] = d ? 1.0f / d : 0.0f; + + y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d); + } + + for (int j = 0; j < 8; j++) { + /* Uses non-default rounding for vec_signed or vec_round */ + const int32x4_t v_qs0 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[0][j], id[0]), 4, 1)); + const int32x4_t v_qs1 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[1][j], id[1]), 4, 1)); + const int32x4_t v_qs2 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[2][j], id[2]), 4, 1)); + const int32x4_t v_qs3 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[3][j], id[3]), 4, 1)); + + const int16x8_t v_qs01 = vec_packs(v_qs0, v_qs1); + const int16x8_t v_qs23 = vec_packs(v_qs2, v_qs3); + + vec_xst(vec_packs(v_qs01, v_qs23), 0, y[i].qs + 16 * j); + } + } +#else + UNUSED(nb); + UNUSED(y); + ggml_quantize_mat_q8_0_4x4_generic(x, vy, k); +#endif +} + +#if defined(__VXE__) || defined(__VXE2__) +static inline int16x8_t vxe_dot_acc(const int8x16_t v_x, const int8x16_t v_y, const int16x8_t v_acc) { + return vec_meadd(v_x, v_y, vec_moadd(v_x, v_y, v_acc)); +} + +static inline int8x16_t vxe_splat_granule(const int8_t * qs) { + uint32_t g; + memcpy(&g, qs, sizeof(g)); + return (int8x16_t)vec_splats(g); +} + +static inline int32x4_t vxe_fold(const int16x8_t v_sumi) { + const int16x8_t v_ones = vec_splats((int16_t)1); + return vec_add(vec_mule(v_sumi, v_ones), vec_mulo(v_sumi, v_ones)); +} +#endif + +void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int qk = QK8_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(nr == 1); + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(bs); + UNUSED(nr); + +#if defined(__VXE__) || defined(__VXE2__) + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + float * res_ptr = s; + + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb); + + float32x4_t v_sumf = vec_splats(0.0f); + + for (int l = 0; l < nb; l++) { + const int8_t * x_qs = b_ptr[l].qs; + + const int8x16_t v_x0 = vec_xl( 0, x_qs); + const int8x16_t v_x1 = vec_xl(16, x_qs); + const int8x16_t v_x2 = vec_xl(32, x_qs); + const int8x16_t v_x3 = vec_xl(48, x_qs); + + const int8x16_t v_x0l = vec_sra(vec_sl(v_x0, 4), 4); + const int8x16_t v_x1l = vec_sra(vec_sl(v_x1, 4), 4); + const int8x16_t v_x2l = vec_sra(vec_sl(v_x2, 4), 4); + const int8x16_t v_x3l = vec_sra(vec_sl(v_x3, 4), 4); + + const int8x16_t v_x0h = vec_sra(v_x0, 4); + const int8x16_t v_x1h = vec_sra(v_x1, 4); + const int8x16_t v_x2h = vec_sra(v_x2, 4); + const int8x16_t v_x3h = vec_sra(v_x3, 4); + + const int8_t * y_lo = a_ptr[l].qs; + const int8_t * y_hi = y_lo + qk / 2; + + int16x8_t v_sumi = vec_splats((int16_t)0); + + v_sumi = vxe_dot_acc(v_x0l, vxe_splat_granule(y_lo + 0), v_sumi); + v_sumi = vxe_dot_acc(v_x1l, vxe_splat_granule(y_lo + 4), v_sumi); + v_sumi = vxe_dot_acc(v_x2l, vxe_splat_granule(y_lo + 8), v_sumi); + v_sumi = vxe_dot_acc(v_x3l, vxe_splat_granule(y_lo + 12), v_sumi); + + v_sumi = vxe_dot_acc(v_x0h, vxe_splat_granule(y_hi + 0), v_sumi); + v_sumi = vxe_dot_acc(v_x1h, vxe_splat_granule(y_hi + 4), v_sumi); + v_sumi = vxe_dot_acc(v_x2h, vxe_splat_granule(y_hi + 8), v_sumi); + v_sumi = vxe_dot_acc(v_x3h, vxe_splat_granule(y_hi + 12), v_sumi); + + const float32x4_t v_yd = vec_splats(GGML_CPU_FP16_TO_FP32(a_ptr[l].d)); + const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d); + const float32x4_t v_d = vec_mul(v_yd, v_xd); + + v_sumf = vec_madd(vec_float(vxe_fold(v_sumi)), v_d, v_sumf); + } + + vec_xst(v_sumf, 0, res_ptr + x * ncols_interleaved); + } +#else + UNUSED(nb); + UNUSED(ncols_interleaved); + ggml_gemv_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +#endif +} + +void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int qk = QK8_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(nr % 4 == 0); + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + +#if defined(__VXE__) || defined(__VXE2__) + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb); + + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb); + + float32x4_t v_sumf[4]; + for (int m = 0; m < 4; m++) { + v_sumf[m] = vec_splats(0.0f); + } + + for (int l = 0; l < nb; l++) { + int16x8_t v_sumi0 = vec_splats((int16_t)0); + int16x8_t v_sumi1 = vec_splats((int16_t)0); + int16x8_t v_sumi2 = vec_splats((int16_t)0); + int16x8_t v_sumi3 = vec_splats((int16_t)0); + + for (int k = 0; k < 4; k++) { + const int8x16_t v_x = vec_xl(0, b_ptr[l].qs + 16 * k); + const int8x16_t v_xl = vec_sra(vec_sl(v_x, 4), 4); + const int8x16_t v_xh = vec_sra(v_x, 4); + + const int8_t * y_lo = a_ptr[l].qs + 16 * k; + const int8_t * y_hi = y_lo + qk / 2 * 4; + + v_sumi0 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 0), v_sumi0); + v_sumi1 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 4), v_sumi1); + v_sumi2 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 8), v_sumi2); + v_sumi3 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 12), v_sumi3); + + v_sumi0 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 0), v_sumi0); + v_sumi1 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 4), v_sumi1); + v_sumi2 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 8), v_sumi2); + v_sumi3 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 12), v_sumi3); + } + + const float32x4_t v_yd = __lzs_f16cx4_load(a_ptr[l].d); + const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d); + + v_sumf[0] = vec_madd(vec_float(vxe_fold(v_sumi0)), vec_mul(v_xd, vec_splat(v_yd, 0)), v_sumf[0]); + v_sumf[1] = vec_madd(vec_float(vxe_fold(v_sumi1)), vec_mul(v_xd, vec_splat(v_yd, 1)), v_sumf[1]); + v_sumf[2] = vec_madd(vec_float(vxe_fold(v_sumi2)), vec_mul(v_xd, vec_splat(v_yd, 2)), v_sumf[2]); + v_sumf[3] = vec_madd(vec_float(vxe_fold(v_sumi3)), vec_mul(v_xd, vec_splat(v_yd, 3)), v_sumf[3]); + } + + for (int m = 0; m < 4; m++) { + vec_xst(v_sumf[m], 0, s + (y * 4 + m) * bs + x * ncols_interleaved); + } + } + } +#else + UNUSED(nb); + UNUSED(ncols_interleaved); + ggml_gemm_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +#endif +} diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 5d1ca5ffcc36..5dd9ec8e628a 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -78,7 +78,7 @@ struct ggml_compute_params { #if defined(__ARM_NEON) // ref: https://github.com/ggml-org/llama.cpp/pull/5404 -#ifdef _MSC_VER +#if defined(_MSC_VER) && !defined(__clang__) #define ggml_vld1q_u32(w,x,y,z) { ((w) + ((uint64_t)(x) << 32)), ((y) + ((uint64_t)(z) << 32)) } #else #define ggml_vld1q_u32(w,x,y,z) { (w), (x), (y), (z) } diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index a4f296c3b463..f9367061fcf6 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -1423,7 +1423,9 @@ UseGgmlGemm1:; const size_t nbw3 = nbw2*ne12; assert(params->wsize >= ne13*nbw3); - GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); + // the F16 path below writes plain floats into wdata, so it needs an F32 vec_dot_type + GGML_ASSERT(src1->type == GGML_TYPE_F32 || vec_dot_type == GGML_TYPE_F32); #if 0 for (int64_t i13 = 0; i13 < ne13; ++i13) { @@ -1442,9 +1444,15 @@ UseGgmlGemm1:; size_t bs = ggml_blck_size(vec_dot_type); int64_t ne10_block_start = (ith * ne10/bs) / nth; int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth; - from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10), - (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0), - (ne10_block_end - ne10_block_start) * bs); + const char * src1_block = (const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10; + char * dst_block = wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0; + const int64_t n_block = (ne10_block_end - ne10_block_start) * bs; + + if (src1->type == GGML_TYPE_F32) { + from_float((const float *) src1_block, dst_block, n_block); + } else { + ggml_cpu_fp16_to_fp32((const ggml_fp16_t *) src1_block, (float *) dst_block, n_block); + } } } } diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 8cece71f186f..1df0f2bb9268 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -451,6 +451,10 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st op->type != GGML_TYPE_IQ1_S && op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float case GGML_OP_MUL_MAT: + if (ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD && + src0->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) { + return src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16; + } return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type; case GGML_OP_SOFT_MAX_BACK: { if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) { diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 106458fa0a4a..acd2c6e0f9ed 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -11529,10 +11529,19 @@ static void ggml_compute_forward_dsv4_hc_pre_f32( const int64_t hc = x->ne[1]; const int64_t n_tokens = x->ne[2]; + const float scale = ggml_get_op_params_f32(dst, 0); + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + GGML_ASSERT(dst->ne[0] == n_embd); GGML_ASSERT(dst->ne[1] == n_tokens); - GGML_ASSERT(weights->ne[0] == hc); - GGML_ASSERT(weights->ne[1] == n_tokens); + if (gated) { + GGML_ASSERT(weights->ne[0] == n_embd); + GGML_ASSERT(weights->ne[1] == hc); + GGML_ASSERT(weights->ne[2] == n_tokens); + } else { + GGML_ASSERT(weights->ne[0] == hc); + GGML_ASSERT(weights->ne[1] == n_tokens); + } GGML_TENSOR_LOCALS(size_t, nbx, x, nb); GGML_TENSOR_LOCALS(size_t, nbw, weights, nb); @@ -11552,12 +11561,18 @@ static void ggml_compute_forward_dsv4_hc_pre_f32( float sum = 0.0f; for (int64_t ih = 0; ih < hc; ++ih) { - const float xv = *(const float *) ((const char *) x->data + i0*nbx0 + ih*nbx1 + it*nbx2); - const float wv = *(const float *) ((const char *) weights->data + ih*nbw0 + it*nbw1); + const float xv = *(const float *) ((const char *) x->data + i0*nbx0 + ih*nbx1 + it*nbx2); + float wv; + if (gated) { + const float gv = *(const float *) ((const char *) weights->data + i0*nbw0 + ih*nbw1 + it*nbw2); + wv = 1.0f / (1.0f + expf(-gv)); + } else { + wv = *(const float *) ((const char *) weights->data + ih*nbw0 + it*nbw1); + } sum += xv * wv; } - *(float *) ((char *) dst->data + i0*nbd0 + it*nbd1) = sum; + *(float *) ((char *) dst->data + i0*nbd0 + it*nbd1) = scale * sum; } } @@ -11591,7 +11606,6 @@ static void ggml_compute_forward_dsv4_hc_post_f32( GGML_ASSERT(x->type == GGML_TYPE_F32); GGML_ASSERT(residual->type == GGML_TYPE_F32); GGML_ASSERT(post->type == GGML_TYPE_F32); - GGML_ASSERT(comb->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); const int64_t n_embd = x->ne[0]; @@ -11605,14 +11619,24 @@ static void ggml_compute_forward_dsv4_hc_post_f32( GGML_ASSERT(residual->ne[2] == n_tokens); GGML_ASSERT(post->ne[0] == hc); GGML_ASSERT(post->ne[1] == n_tokens); - GGML_ASSERT(comb->ne[0] == hc); - GGML_ASSERT(comb->ne[1] == hc); - GGML_ASSERT(comb->ne[2] == n_tokens); + + // comb == NULL: identity mixing, each stream keeps its own residual + size_t nbc0 = 0; + size_t nbc1 = 0; + size_t nbc2 = 0; + if (comb) { + GGML_ASSERT(comb->type == GGML_TYPE_F32); + GGML_ASSERT(comb->ne[0] == hc); + GGML_ASSERT(comb->ne[1] == hc); + GGML_ASSERT(comb->ne[2] == n_tokens); + nbc0 = comb->nb[0]; + nbc1 = comb->nb[1]; + nbc2 = comb->nb[2]; + } GGML_TENSOR_LOCALS(size_t, nbx, x, nb); GGML_TENSOR_LOCALS(size_t, nbr, residual, nb); GGML_TENSOR_LOCALS(size_t, nbp, post, nb); - GGML_TENSOR_LOCALS(size_t, nbc, comb, nb); GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); const int ith = params->ith; @@ -11632,10 +11656,14 @@ static void ggml_compute_forward_dsv4_hc_post_f32( const float pv = *(const float *) ((const char *) post->data + idst*nbp0 + it*nbp1); float sum = xv * pv; - for (int64_t isrc = 0; isrc < hc; ++isrc) { - const float rv = *(const float *) ((const char *) residual->data + i0*nbr0 + isrc*nbr1 + it*nbr2); - const float cv = *(const float *) ((const char *) comb->data + idst*nbc0 + isrc*nbc1 + it*nbc2); - sum += rv * cv; + if (comb) { + for (int64_t isrc = 0; isrc < hc; ++isrc) { + const float rv = *(const float *) ((const char *) residual->data + i0*nbr0 + isrc*nbr1 + it*nbr2); + const float cv = *(const float *) ((const char *) comb->data + idst*nbc0 + isrc*nbc1 + it*nbc2); + sum += rv * cv; + } + } else { + sum += *(const float *) ((const char *) residual->data + i0*nbr0 + idst*nbr1 + it*nbr2); } *(float *) ((char *) dst->data + i0*nbd0 + idst*nbd1 + it*nbd2) = sum; @@ -12979,11 +13007,20 @@ void ggml_compute_forward_fp8_mul_mat(const ggml_compute_params * params, ggml_t free(w_dec); } -static void ggml_compute_forward_fwht_f32(const ggml_compute_params * params, ggml_tensor * dst) { +static inline float ggml_fwht_load(const float value) { + return value; +} + +static inline float ggml_fwht_load(const ggml_fp16_t value) { + return ggml_fp16_to_fp32(value); +} + +template<typename src_t> +static void ggml_compute_forward_fwht_impl(const ggml_compute_params * params, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == (std::is_same_v<src_t, float> ? GGML_TYPE_F32 : GGML_TYPE_F16)); GGML_ASSERT(dst->type == GGML_TYPE_F32); GGML_TENSOR_BINARY_OP_LOCALS @@ -13010,11 +13047,11 @@ static void ggml_compute_forward_fwht_f32(const ggml_compute_params * params, gg const int64_t i12 = (r - i13 * ne11 * ne12) / ne11; const int64_t i11 = r - i13 * ne11 * ne12 - i12 * ne11; - const float * src_row = (const float *) ((const char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13); + const src_t * src_row = (const src_t *) ((const char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13); float * dst_row = (float *) ((char *) dst->data + i11 * nb1 + i12 * nb2 + i13 * nb3); for (int64_t j = 0; j < n; j++) { - dst_row[j] = src_row[j] * scale; + dst_row[j] = ggml_fwht_load(src_row[j]) * scale; } // Scalar passes @@ -13061,12 +13098,17 @@ void ggml_compute_forward_fwht(const ggml_compute_params * params, ggml_tensor * switch (src1->type) { case GGML_TYPE_F32: { - ggml_compute_forward_fwht_f32(params, dst); + ggml_compute_forward_fwht_impl<float>(params, dst); + } + break; + case GGML_TYPE_F16: + { + ggml_compute_forward_fwht_impl<ggml_fp16_t>(params, dst); } break; default: { - GGML_ABORT("fatal error - fwht is F32 only"); + GGML_ABORT("fatal error - fwht supports F32 and F16 input"); } } } diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h index dc55340dcd9f..3de310e9f1df 100644 --- a/ggml/src/ggml-cpu/ops.h +++ b/ggml/src/ggml-cpu/ops.h @@ -5,10 +5,10 @@ // // cache line // - -#if defined(__cpp_lib_hardware_interference_size) -#define CACHE_LINE_SIZE std::hardware_destructive_interference_size -#else +// TODO: rework CACHE_LINE_SIZE so std::hardware_destructive_interference_size +// can be used consistently between C and C++ TUs; the previous macro form +// diverged based on include order and undersized the work buffer. +// ref: https://github.com/ggml-org/llama.cpp/pull/28882 #if defined(__POWER9_VECTOR__) #define CACHE_LINE_SIZE 128 #elif defined(__VXE__) || defined(__VXE2__) @@ -16,7 +16,6 @@ #else #define CACHE_LINE_SIZE 64 #endif -#endif static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float); diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index 9689ca3ced8f..d56db98023b7 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -1365,6 +1365,133 @@ void ggml_gemv_q8_0_4x8_q8_0_generic(int n, } } +void ggml_gemv_q1_0_4x4_q8_0_generic(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(nr == 1); + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(bs); + UNUSED(nr); + + float sumf[4]; + + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + for (int j = 0; j < ncols_interleaved; j++) { + sumf[j] = 0.0; + } + + for (int l = 0; l < nb; l++) { + const float d0[4] = { + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]), + }; + + for (int k = 0; k < QK1_0 / QK8_0; ++k) { + const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * (QK1_0 / QK8_0) + k; + const float d1 = GGML_CPU_FP16_TO_FP32(a_blk->d); + const float scale[4] = { d0[0] * d1, d0[1] * d1, d0[2] * d1, d0[3] * d1 }; + + for (int tile = 0; tile < QK8_0 / 4; ++tile) { + const uint8_t bits_lo = b_ptr[l].qs[k * 16 + 2 * tile + 0]; + const uint8_t bits_hi = b_ptr[l].qs[k * 16 + 2 * tile + 1]; + + for (int p = 0; p < 4; ++p) { + const float q = (float) a_blk->qs[tile * 4 + p]; + + sumf[0] += ((bits_lo & (1u << p)) ? scale[0] : -scale[0]) * q; + sumf[1] += ((bits_lo & (1u << (4 + p))) ? scale[1] : -scale[1]) * q; + sumf[2] += ((bits_hi & (1u << p)) ? scale[2] : -scale[2]) * q; + sumf[3] += ((bits_hi & (1u << (4 + p))) ? scale[3] : -scale[3]) * q; + } + } + } + } + + for (int j = 0; j < ncols_interleaved; j++) { + s[x * ncols_interleaved + j] = sumf[j]; + } + } +} + +void ggml_gemv_q1_0_4x8_q8_0_generic(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + const int blocklen = 8; + + assert(nr == 1); + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(bs); + UNUSED(nr); + + float sumf[4]; + + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + for (int j = 0; j < ncols_interleaved; j++) { + sumf[j] = 0.0f; + } + + for (int l = 0; l < nb; l++) { + const float d0[4] = { + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]), + }; + + for (int k = 0; k < qk / blocklen; ++k) { + const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * (qk / QK8_0) + k / (QK8_0 / blocklen); + const float d1 = GGML_CPU_FP16_TO_FP32(a_blk->d); + const float scale[4] = { d0[0] * d1, d0[1] * d1, d0[2] * d1, d0[3] * d1 }; + const uint8_t bits0 = b_ptr[l].qs[k * ncols_interleaved + 0]; + const uint8_t bits1 = b_ptr[l].qs[k * ncols_interleaved + 1]; + const uint8_t bits2 = b_ptr[l].qs[k * ncols_interleaved + 2]; + const uint8_t bits3 = b_ptr[l].qs[k * ncols_interleaved + 3]; + const int q_offset = (k % (QK8_0 / blocklen)) * blocklen; + + for (int p = 0; p < blocklen; ++p) { + const float q = (float) a_blk->qs[q_offset + p]; + + sumf[0] += ((bits0 & (1u << p)) ? scale[0] : -scale[0]) * q; + sumf[1] += ((bits1 & (1u << p)) ? scale[1] : -scale[1]) * q; + sumf[2] += ((bits2 & (1u << p)) ? scale[2] : -scale[2]) * q; + sumf[3] += ((bits3 & (1u << p)) ? scale[3] : -scale[3]) * q; + } + } + } + + for (int j = 0; j < ncols_interleaved; j++) { + s[x * ncols_interleaved + j] = sumf[j]; + } + } +} + // Only enable these for RISC-V. #if defined __riscv_zvfh void ggml_gemv_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { @@ -2383,6 +2510,176 @@ void ggml_gemm_q8_0_4x8_q8_0_generic(int n, } } +void ggml_gemm_q1_0_4x4_q8_0_generic(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(n % qk == 0); + assert(nr % 4 == 0); + assert(nc % ncols_interleaved == 0); + + float sumf[4][4]; + + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + sumf[m][j] = 0.0; + } + } + + for (int l = 0; l < nb; l++) { + const float d0[4] = { + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]), + }; + + for (int k = 0; k < QK1_0 / QK8_0; ++k) { + const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k; + const float a_d[4] = { + GGML_CPU_FP16_TO_FP32(a_blk->d[0]), + GGML_CPU_FP16_TO_FP32(a_blk->d[1]), + GGML_CPU_FP16_TO_FP32(a_blk->d[2]), + GGML_CPU_FP16_TO_FP32(a_blk->d[3]), + }; + + for (int tile = 0; tile < QK8_0 / 4; ++tile) { + const uint8_t bits_lo = b_ptr[l].qs[k * 16 + 2 * tile + 0]; + const uint8_t bits_hi = b_ptr[l].qs[k * 16 + 2 * tile + 1]; + const int tile_offset = tile * 16; + + for (int p = 0; p < 4; ++p) { + const int8_t q_row[4] = { + a_blk->qs[tile_offset + 0 * 4 + p], + a_blk->qs[tile_offset + 1 * 4 + p], + a_blk->qs[tile_offset + 2 * 4 + p], + a_blk->qs[tile_offset + 3 * 4 + p], + }; + const int sign[4] = { + (bits_lo & (1u << p)) ? 1 : -1, + (bits_lo & (1u << (4 + p))) ? 1 : -1, + (bits_hi & (1u << p)) ? 1 : -1, + (bits_hi & (1u << (4 + p))) ? 1 : -1, + }; + + for (int m = 0; m < 4; ++m) { + const float row_scale = a_d[m]; + sumf[m][0] += sign[0] * q_row[m] * d0[0] * row_scale; + sumf[m][1] += sign[1] * q_row[m] * d0[1] * row_scale; + sumf[m][2] += sign[2] * q_row[m] * d0[2] * row_scale; + sumf[m][3] += sign[3] * q_row[m] * d0[3] * row_scale; + } + } + } + } + } + + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j]; + } + } + } + } +} + +void ggml_gemm_q1_0_4x8_q8_0_generic(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + const int blocklen = 8; + + assert(n % qk == 0); + assert(nr % 4 == 0); + assert(nc % ncols_interleaved == 0); + + float sumf[4][4]; + + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + sumf[m][j] = 0.0f; + } + } + + for (int l = 0; l < nb; l++) { + const float d0[4] = { + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]), + }; + + for (int k = 0; k < qk / blocklen; ++k) { + const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k / (QK8_0 / blocklen); + const float a_d[4] = { + GGML_CPU_FP16_TO_FP32(a_blk->d[0]), + GGML_CPU_FP16_TO_FP32(a_blk->d[1]), + GGML_CPU_FP16_TO_FP32(a_blk->d[2]), + GGML_CPU_FP16_TO_FP32(a_blk->d[3]), + }; + const uint8_t bits0 = b_ptr[l].qs[k * ncols_interleaved + 0]; + const uint8_t bits1 = b_ptr[l].qs[k * ncols_interleaved + 1]; + const uint8_t bits2 = b_ptr[l].qs[k * ncols_interleaved + 2]; + const uint8_t bits3 = b_ptr[l].qs[k * ncols_interleaved + 3]; + const int q_offset = (k % (QK8_0 / blocklen)) * 4 * blocklen; + + for (int p = 0; p < blocklen; ++p) { + const int8_t q_row[4] = { + a_blk->qs[q_offset + 0 * blocklen + p], + a_blk->qs[q_offset + 1 * blocklen + p], + a_blk->qs[q_offset + 2 * blocklen + p], + a_blk->qs[q_offset + 3 * blocklen + p], + }; + const int sign[4] = { + (bits0 & (1u << p)) ? 1 : -1, + (bits1 & (1u << p)) ? 1 : -1, + (bits2 & (1u << p)) ? 1 : -1, + (bits3 & (1u << p)) ? 1 : -1, + }; + + for (int m = 0; m < 4; ++m) { + const float row_scale = a_d[m]; + sumf[m][0] += sign[0] * q_row[m] * d0[0] * row_scale; + sumf[m][1] += sign[1] * q_row[m] * d0[1] * row_scale; + sumf[m][2] += sign[2] * q_row[m] * d0[2] * row_scale; + sumf[m][3] += sign[3] * q_row[m] * d0[3] * row_scale; + } + } + } + } + + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j]; + } + } + } + } +} + // Only enable these for RISC-V. #if defined __riscv_zvfh void ggml_gemm_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { @@ -2739,6 +3036,50 @@ static block_q8_0x4 make_block_q8_0x4(block_q8_0 * in, unsigned int blck_size_in return out; } +static block_q1_0x4 make_block_q1_0x4(block_q1_0 * in, unsigned int blck_size_interleave) { + block_q1_0x4 out; + + for (int i = 0; i < 4; i++) { + out.d[i] = in[i].d; + } + + GGML_ASSERT(blck_size_interleave == 4 || blck_size_interleave == 8); + + if (blck_size_interleave == 4) { + for (int k = 0; k < QK1_0 / QK8_0; ++k) { + for (int tile = 0; tile < QK8_0 / 4; ++tile) { + uint8_t packed_lo = 0; + uint8_t packed_hi = 0; + + const int weight_base = k * QK8_0 + tile * 4; + for (int pos = 0; pos < 4; ++pos) { + const int weight_idx = weight_base + pos; + const int byte_idx = weight_idx / 8; + const int bit_idx = weight_idx % 8; + + packed_lo |= ((in[0].qs[byte_idx] >> bit_idx) & 1u) << pos; + packed_lo |= ((in[1].qs[byte_idx] >> bit_idx) & 1u) << (4 + pos); + packed_hi |= ((in[2].qs[byte_idx] >> bit_idx) & 1u) << pos; + packed_hi |= ((in[3].qs[byte_idx] >> bit_idx) & 1u) << (4 + pos); + } + + out.qs[k * 16 + 2 * tile + 0] = packed_lo; + out.qs[k * 16 + 2 * tile + 1] = packed_hi; + } + } + return out; + } + + for (int byte_idx = 0; byte_idx < QK1_0 / 8; ++byte_idx) { + out.qs[byte_idx * 4 + 0] = in[0].qs[byte_idx]; + out.qs[byte_idx * 4 + 1] = in[1].qs[byte_idx]; + out.qs[byte_idx * 4 + 2] = in[2].qs[byte_idx]; + out.qs[byte_idx * 4 + 3] = in[3].qs[byte_idx]; + } + + return out; +} + static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, int blck_size_interleave) { block_q4_0x4 out; @@ -3509,6 +3850,38 @@ static int repack_q8_0_to_q8_0_4_bl(struct ggml_tensor * t, return 0; } +static int repack_q1_0_to_q1_0_4_bl(struct ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q1_0); + GGML_ASSERT(interleave_block == 4 || interleave_block == 8); + constexpr int nrows_interleaved = 4; + + block_q1_0x4 * dst = (block_q1_0x4 *) t->data; + const block_q1_0 * src = (const block_q1_0 *) data; + block_q1_0 dst_tmp[4]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK1_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q1_0)); + + if (t->ne[1] % nrows_interleaved != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + dst_tmp[i] = src[x + i * nblocks]; + } + *dst++ = make_block_q1_0x4(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; +} + static block_q8_0x16 make_block_q8_0x16(block_q8_0 * in, unsigned int blck_size_interleave) { block_q8_0x16 out; @@ -3865,6 +4238,14 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS> int repack(struct ggml_tensor *, const void *, size_t); // TODO: generalise. +template <> int repack<block_q1_0, 4, 4>(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_q1_0_to_q1_0_4_bl(t, 4, data, data_size); +} + +template <> int repack<block_q1_0, 8, 4>(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_q1_0_to_q1_0_4_bl(t, 8, data, data_size); +} + template <> int repack<block_q4_0, 4, 4>(struct ggml_tensor * t, const void * data, size_t data_size) { return repack_q4_0_to_q4_0_4_bl(t, 4, data, data_size); } @@ -3960,6 +4341,14 @@ template <> int repack<block_q2_K, 1, 16>(struct ggml_tensor * t, const void * d template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PARAM_TYPE> void gemv(int, float *, size_t, const void *, const void *, int, int); +template <> void gemv<block_q1_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_q1_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemv<block_q1_0, 8, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_q1_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); +} + template <> void gemv<block_q4_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemv_q4_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc); } @@ -4057,6 +4446,14 @@ template <> void gemv<block_q2_K, 1, 16, GGML_TYPE_Q8_K>(int n, float * s, size_ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PARAM_TYPE> void gemm(int, float *, size_t, const void *, const void *, int, int); +template <> void gemm<block_q1_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_q1_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemm<block_q1_0, 8, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_q1_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); +} + template <> void gemm<block_q4_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_q4_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc); } @@ -4526,6 +4923,10 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PAR } // namespace ggml::cpu::repack static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(const struct ggml_tensor * cur) { + // instance for Q1_0 + static const ggml::cpu::repack::tensor_traits<block_q1_0, 4, 4, GGML_TYPE_Q8_0> q1_0_4x4_q8_0; + static const ggml::cpu::repack::tensor_traits<block_q1_0, 8, 4, GGML_TYPE_Q8_0> q1_0_4x8_q8_0; + // instance for Q4 static const ggml::cpu::repack::tensor_traits<block_q4_0, 4, 4, GGML_TYPE_Q8_0> q4_0_4x4_q8_0; static const ggml::cpu::repack::tensor_traits<block_q4_0, 8, 4, GGML_TYPE_Q8_0> q4_0_4x8_q8_0; @@ -4586,6 +4987,11 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons return &q4_0_4x4_q8_0; } } + if (ggml_cpu_has_vxe()) { + if (cur->ne[1] % 4 == 0) { + return &q4_0_4x4_q8_0; + } + } if (ggml_cpu_has_riscv_v()) { #if defined __riscv_zvfh switch (__riscv_vlenb() * 8) { @@ -4718,6 +5124,17 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons } #endif } + } else if (cur->type == GGML_TYPE_Q1_0) { + if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) { + if (cur->ne[1] % 4 == 0) { + return &q1_0_4x8_q8_0; + } + } + if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) { + if (cur->ne[1] % 4 == 0) { + return &q1_0_4x4_q8_0; + } + } } return nullptr; diff --git a/ggml/src/ggml-cpu/repack.h b/ggml/src/ggml-cpu/repack.h index cb21edf62394..fc6715c39ce9 100644 --- a/ggml/src/ggml-cpu/repack.h +++ b/ggml/src/ggml-cpu/repack.h @@ -11,6 +11,9 @@ ggml_backend_buffer_type_t ggml_backend_cpu_repack_buffer_type(void); template <int K> constexpr int QK_0() { + if constexpr (K == 1) { + return QK1_0; + } if constexpr (K == 4) { return QK4_0; } @@ -26,6 +29,7 @@ template <int K, int N> struct block { }; // control size +static_assert(sizeof(block<1, 4>) == 4 * sizeof(ggml_half) + QK1_0 / 2, "wrong block<1,4> size/padding"); static_assert(sizeof(block<4, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 2, "wrong block<4,4> size/padding"); static_assert(sizeof(block<4, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 4, "wrong block<4,8> size/padding"); static_assert(sizeof(block<4, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 8, "wrong block<4,16> size/padding"); @@ -33,6 +37,7 @@ static_assert(sizeof(block<8, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 4, "wrong b static_assert(sizeof(block<8, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 8, "wrong block<8,8> size/padding"); static_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 16, "wrong block<8,16> size/padding"); +using block_q1_0x4 = block<1, 4>; using block_q4_0x4 = block<4, 4>; using block_q4_0x8 = block<4, 8>; using block_q4_0x16 = block<4, 16>; @@ -141,6 +146,8 @@ void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTR void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_gemv_q1_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_q1_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -157,6 +164,8 @@ void ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v void ggml_gemv_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -193,6 +202,8 @@ void ggml_quantize_mat_q8_0_4x4_generic(const float * GGML_RESTRICT x, void * GG void ggml_quantize_mat_q8_0_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_gemv_q1_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_q1_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -209,6 +220,8 @@ void ggml_gemv_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_gemv_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); diff --git a/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp b/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp index d2f897436220..13b84dcbe72b 100644 --- a/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +++ b/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp @@ -639,7 +639,7 @@ static void permute_transpose_impl(const ggml_tensor * src0, } } else if (n_src_stride == sizeof(int16_t)) { for (int64_t bi = ith; bi < batch; bi += nth) { - rvv_transposed_s32_mn_to_nm((int8_t *) ((char *) dst->data + bi * batch_stride), n_dst_stride, + rvv_transposed_s16_mn_to_nm((int8_t *) ((char *) dst->data + bi * batch_stride), n_dst_stride, (int8_t *) ((char *) src0->data + bi * batch_stride), m_src_stride, m, n); } } else { diff --git a/ggml/src/ggml-cuda/CMakeLists.txt b/ggml/src/ggml-cuda/CMakeLists.txt index f486f940def9..0a9701d5edb5 100644 --- a/ggml/src/ggml-cuda/CMakeLists.txt +++ b/ggml/src/ggml-cuda/CMakeLists.txt @@ -118,32 +118,8 @@ if (CUDAToolkit_FOUND) file(GLOB SRCS "template-instances/mmf*.cu") list(APPEND GGML_SOURCES_CUDA ${SRCS}) - if (GGML_CUDA_FA_ALL_QUANTS) - file(GLOB SRCS "template-instances/fattn-vec*.cu") - list(APPEND GGML_SOURCES_CUDA ${SRCS}) - add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS) - else() - list(APPEND GGML_SOURCES_CUDA - template-instances/fattn-vec-instance-f16-f16.cu - template-instances/fattn-vec-instance-q4_0-q4_0.cu - template-instances/fattn-vec-instance-q8_0-q8_0.cu - template-instances/fattn-vec-instance-bf16-bf16.cu - template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu - template-instances/fattn-vec-instance-turbo3_0-q8_0.cu - template-instances/fattn-vec-instance-q8_0-turbo3_0.cu - template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu - template-instances/fattn-vec-instance-turbo2_0-q8_0.cu - template-instances/fattn-vec-instance-q8_0-turbo2_0.cu - template-instances/fattn-vec-instance-turbo3_0-turbo2_0.cu - template-instances/fattn-vec-instance-turbo2_0-turbo3_0.cu - template-instances/fattn-vec-instance-turbo4_0-turbo4_0.cu - template-instances/fattn-vec-instance-turbo4_0-q8_0.cu - template-instances/fattn-vec-instance-q8_0-turbo4_0.cu - template-instances/fattn-vec-instance-turbo4_0-turbo3_0.cu - template-instances/fattn-vec-instance-turbo3_0-turbo4_0.cu - template-instances/fattn-vec-instance-turbo4_0-turbo2_0.cu - template-instances/fattn-vec-instance-turbo2_0-turbo4_0.cu) - endif() + ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR} SRCS) + list(APPEND GGML_SOURCES_CUDA ${SRCS}) ggml_add_backend_library(ggml-cuda ${GGML_HEADERS_CUDA} diff --git a/ggml/src/ggml-cuda/allreduce.cu b/ggml/src/ggml-cuda/allreduce.cu index b237ddd6dfc3..5b81018dff9a 100644 --- a/ggml/src/ggml-cuda/allreduce.cu +++ b/ggml/src/ggml-cuda/allreduce.cu @@ -1308,7 +1308,7 @@ struct ggml_cuda_ar_host_mapping { } rc = cudaHostGetDevicePointer(reinterpret_cast<void **>(&dev), host, 0); if (rc != cudaSuccess) { - cudaFreeHost(host); + CUDA_CHECK(cudaFreeHost(host)); host = nullptr; dev = nullptr; } @@ -1317,7 +1317,7 @@ struct ggml_cuda_ar_host_mapping { void free() { if (host) { - cudaFreeHost(host); + CUDA_CHECK(cudaFreeHost(host)); host = nullptr; dev = nullptr; } @@ -2707,28 +2707,28 @@ void ggml_cuda_ar_pipeline_free(ggml_cuda_ar_pipeline * p) { p->spin_dbg[i].free(); if (p->dev_tmp[i]) { ggml_cuda_set_device(p->devices[i]); - cudaFree(p->dev_tmp[i]); + CUDA_CHECK(cudaFree(p->dev_tmp[i])); } ggml_cuda_set_device(p->devices[i]); for (int s = 0; s < GGML_CUDA_AR_POOL_SIZE; ++s) { - if (p->ev_pool[i][s].app) { cudaEventDestroy(p->ev_pool[i][s].app); } + if (p->ev_pool[i][s].app) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].app)); } for (int c = 0; c < GGML_CUDA_AR_COPY_MAX_CHUNKS; ++c) { - if (p->ev_pool[i][s].cpy[c]) { cudaEventDestroy(p->ev_pool[i][s].cpy[c]); } + if (p->ev_pool[i][s].cpy[c]) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].cpy[c])); } } - if (p->ev_pool[i][s].h2d) { cudaEventDestroy(p->ev_pool[i][s].h2d); } - if (p->ev_pool[i][s].ker) { cudaEventDestroy(p->ev_pool[i][s].ker); } + if (p->ev_pool[i][s].h2d) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].h2d)); } + if (p->ev_pool[i][s].ker) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].ker)); } } if (p->host_large_read_done[i]) { ggml_cuda_set_device(p->devices[i]); - cudaEventDestroy(p->host_large_read_done[i]); + CUDA_CHECK(cudaEventDestroy(p->host_large_read_done[i])); } if (p->dev_tmp_kernel_done[i]) { ggml_cuda_set_device(p->devices[i]); - cudaEventDestroy(p->dev_tmp_kernel_done[i]); + CUDA_CHECK(cudaEventDestroy(p->dev_tmp_kernel_done[i])); } if (p->streams[i]) { ggml_cuda_set_device(p->devices[i]); - cudaStreamDestroy(p->streams[i]); + CUDA_CHECK(cudaStreamDestroy(p->streams[i])); } } p->arrival.free(); diff --git a/ggml/src/ggml-cuda/allreduce.cuh b/ggml/src/ggml-cuda/allreduce.cuh index cf09e47446d1..305c7fe1b95b 100644 --- a/ggml/src/ggml-cuda/allreduce.cuh +++ b/ggml/src/ggml-cuda/allreduce.cuh @@ -15,7 +15,7 @@ void ggml_cuda_ar_wait_logged(cudaStream_t stream, cudaEvent_t event, const char struct ggml_cuda_ar_pipeline; // Allocate a pipeline for n_devices GPUs. -// devices[] holds the CUDA device IDs in rank order. +// devices[] holds the GPU device IDs in rank order. // Returns nullptr on allocation failure. ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init( const int * devices, size_t n_devices); diff --git a/ggml/src/ggml-cuda/argsort.cu b/ggml/src/ggml-cuda/argsort.cu index 26af90025972..24115da09960 100644 --- a/ggml/src/ggml-cuda/argsort.cu +++ b/ggml/src/ggml-cuda/argsort.cu @@ -51,9 +51,12 @@ void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, cudaStream_t stream) { ggml_cuda_pool_alloc<int> temp_indices_alloc(pool, ncols * nrows); ggml_cuda_pool_alloc<float> temp_keys_alloc(pool, ncols * nrows); + // Device*Sort algorithms currently do not allow for in-place sorting/aliasing of input/outputs + ggml_cuda_pool_alloc<float> temp_keys_out_alloc(pool, ncols * nrows); int * temp_indices = temp_indices_alloc.get(); float * temp_keys = temp_keys_alloc.get(); + float * temp_keys_out = temp_keys_out_alloc.get(); static const int block_size = 256; const dim3 grid_size((ncols + block_size - 1) / block_size, nrows); @@ -85,18 +88,18 @@ void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, if (order == GGML_SORT_ORDER_ASC) { if (nrows == 1) { - CUDA_CHECK(DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, temp_keys, temp_keys, // keys (in-place) + CUDA_CHECK(DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, temp_keys, temp_keys_out, // keys in, keys out temp_indices, dst, // values (indices) ncols, 0, sizeof(float) * 8, stream)); } else if (is_capturing) { CUDA_CHECK(DeviceSegmentedRadixSort::SortPairs( - nullptr, temp_storage_bytes, temp_keys, temp_keys, // keys (in-place) + nullptr, temp_storage_bytes, temp_keys, temp_keys_out, // keys in, keys out temp_indices, dst, // values (indices) ncols * nrows, nrows, // num items, num segments offset_iterator, offset_iterator + 1, 0, sizeof(float) * 8, stream)); } else { CUDA_CHECK(DeviceSegmentedSort::SortPairs(nullptr, temp_storage_bytes, temp_keys, - temp_keys, // keys (in-place) + temp_keys_out, // keys out temp_indices, dst, // values (indices) ncols * nrows, nrows, // num items, num segments offset_iterator, offset_iterator + 1, stream)); @@ -104,15 +107,15 @@ void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, } else { if (nrows == 1) { CUDA_CHECK(DeviceRadixSort::SortPairsDescending(nullptr, temp_storage_bytes, temp_keys, - temp_keys, // keys (in-place) + temp_keys_out, // keys out temp_indices, dst, // values (indices) ncols, 0, sizeof(float) * 8, stream)); } else if (is_capturing) { CUDA_CHECK(DeviceSegmentedRadixSort::SortPairsDescending( - nullptr, temp_storage_bytes, temp_keys, temp_keys, temp_indices, dst, ncols * nrows, nrows, + nullptr, temp_storage_bytes, temp_keys, temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, 0, sizeof(float) * 8, stream)); } else { - CUDA_CHECK(DeviceSegmentedSort::SortPairsDescending(nullptr, temp_storage_bytes, temp_keys, temp_keys, + CUDA_CHECK(DeviceSegmentedSort::SortPairsDescending(nullptr, temp_storage_bytes, temp_keys, temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, stream)); } @@ -124,31 +127,31 @@ void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, if (order == GGML_SORT_ORDER_ASC) { if (nrows == 1) { CUDA_CHECK(DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, - temp_keys, // keys (in-place) + temp_keys_out, // keys out temp_indices, dst, // values (indices) ncols, 0, sizeof(float) * 8, stream)); } else if (is_capturing) { - CUDA_CHECK(DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys, + CUDA_CHECK(DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, 0, sizeof(float) * 8, stream)); } else { - CUDA_CHECK(DeviceSegmentedSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys, + CUDA_CHECK(DeviceSegmentedSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, stream)); } } else { if (nrows == 1) { CUDA_CHECK(DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, temp_keys, - temp_keys, // keys (in-place) + temp_keys_out, // keys out temp_indices, dst, // values (indices) ncols, 0, sizeof(float) * 8, stream)); } else if (is_capturing) { CUDA_CHECK(DeviceSegmentedRadixSort::SortPairsDescending( - d_temp_storage, temp_storage_bytes, temp_keys, temp_keys, temp_indices, dst, ncols * nrows, nrows, + d_temp_storage, temp_storage_bytes, temp_keys, temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, 0, sizeof(float) * 8, stream)); } else { CUDA_CHECK(DeviceSegmentedSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, temp_keys, - temp_keys, temp_indices, dst, ncols * nrows, nrows, + temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, stream)); } } diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index dbde2e3ae325..93190518b1be 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -84,6 +84,8 @@ #define GGML_CUDA_CC_GCN4 (GGML_CUDA_CC_OFFSET_AMD + 0x803) // Tonga, Fiji, Polaris, minimum for fast fp16 #define GGML_CUDA_CC_VEGA (GGML_CUDA_CC_OFFSET_AMD + 0x900) // Vega56/64, minimum for fp16 dual issue #define GGML_CUDA_CC_VEGA20 (GGML_CUDA_CC_OFFSET_AMD + 0x906) // MI50/Radeon VII, minimum for dp4a +#define GGML_CUDA_CC_GFX909 (GGML_CUDA_CC_OFFSET_AMD + 0x909) // GCN APU +#define GGML_CUDA_CC_GFX90C (GGML_CUDA_CC_OFFSET_AMD + 0x90c) // GCN APU #define GGML_CUDA_CC_CDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x908) // MI100, minimum for MFMA, acc registers #define GGML_CUDA_CC_CDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x90a) // MI210 (gfx90a), minimum acc register renaming #define GGML_CUDA_CC_CDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x942) // MI300 @@ -104,12 +106,13 @@ #define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4) #define GGML_CUDA_CC_IS_RDNA3(cc) (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) #define GGML_CUDA_CC_IS_RDNA4(cc) (cc >= GGML_CUDA_CC_RDNA4) -#define GGML_CUDA_CC_IS_GCN(cc) (cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1) -#define GGML_CUDA_CC_IS_CDNA(cc) (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_RDNA1) -#define GGML_CUDA_CC_IS_CDNA1(cc) (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_CDNA2) -#define GGML_CUDA_CC_IS_CDNA2(cc) (cc >= GGML_CUDA_CC_CDNA2 && cc < GGML_CUDA_CC_CDNA3) -#define GGML_CUDA_CC_IS_CDNA3(cc) (cc >= GGML_CUDA_CC_CDNA3 && cc < GGML_CUDA_CC_CDNA4) -#define GGML_CUDA_CC_IS_CDNA4(cc) (cc >= GGML_CUDA_CC_CDNA4 && cc < GGML_CUDA_CC_RDNA1) +#define GGML_CUDA_CC_IS_GCN_APU(cc) ((cc) == GGML_CUDA_CC_GFX909 || (cc) == GGML_CUDA_CC_GFX90C) +#define GGML_CUDA_CC_IS_GCN(cc) ((cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1) || GGML_CUDA_CC_IS_GCN_APU(cc)) +#define GGML_CUDA_CC_IS_CDNA(cc) (!GGML_CUDA_CC_IS_GCN_APU(cc) && cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_RDNA1) +#define GGML_CUDA_CC_IS_CDNA1(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_CDNA2) +#define GGML_CUDA_CC_IS_CDNA2(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA2 && cc < GGML_CUDA_CC_CDNA3) +#define GGML_CUDA_CC_IS_CDNA3(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA3 && cc < GGML_CUDA_CC_CDNA4) +#define GGML_CUDA_CC_IS_CDNA4(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA4 && cc < GGML_CUDA_CC_RDNA1) // Moore Threads #define MUSART_HMASK 40300 // MUSA rc4.3, min. ver. for half2 -> uint mask comparisons @@ -136,6 +139,12 @@ # define GGML_CUDA_USE_PDL #endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUDART_VERSION >= 12030 || (!(defined(_MSC_VER) && !defined(__clang__)) && CUDART_VERSION >= 11080)) +static __device__ __forceinline__ void ggml_cuda_syncwarp() { +#ifndef GGML_USE_HIP + __syncwarp(); +#endif // GGML_USE_HIP +} + static __device__ __forceinline__ void ggml_cuda_pdl_sync() { #if defined(GGML_CUDA_USE_PDL) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER cudaGridDependencySynchronize(); @@ -335,6 +344,12 @@ static bool fp16_mma_hardware_available(const int cc) { (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2); } +// To be used for feature selection of external libraries, e.g. cuBLAS. +static bool fast_bf16_hardware_available(const int cc) { + return (GGML_CUDA_CC_IS_AMD(cc) && (cc >= GGML_CUDA_CC_RDNA3 || GGML_CUDA_CC_IS_CDNA(cc))) + || (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE); +} + static bool bf16_mma_hardware_available(const int cc) { return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE) || GGML_CUDA_CC_IS_CDNA(cc) || cc >= GGML_CUDA_CC_RDNA3 || @@ -1007,6 +1022,7 @@ template<> struct ggml_cuda_type_traits<GGML_TYPE_F16> { static constexpr int qk = 1; static constexpr int qr = 1; + static constexpr int bs = sizeof(ggml_half); }; template<> @@ -1014,6 +1030,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q1_0> { static constexpr int qk = QK1_0; static constexpr int qr = QR1_0; static constexpr int qi = QI1_0; + static constexpr int bs = sizeof(block_q1_0); }; template<> @@ -1021,6 +1038,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q2_0> { static constexpr int qk = QK2_0; static constexpr int qr = QR2_0; static constexpr int qi = QI2_0; + static constexpr int bs = sizeof(block_q2_0); }; template<> @@ -1028,6 +1046,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q4_0> { static constexpr int qk = QK4_0; static constexpr int qr = QR4_0; static constexpr int qi = QI4_0; + static constexpr int bs = sizeof(block_q4_0); }; template<> @@ -1035,6 +1054,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q4_1> { static constexpr int qk = QK4_1; static constexpr int qr = QR4_1; static constexpr int qi = QI4_1; + static constexpr int bs = sizeof(block_q4_1); }; template<> @@ -1042,6 +1062,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q5_0> { static constexpr int qk = QK5_0; static constexpr int qr = QR5_0; static constexpr int qi = QI5_0; + static constexpr int bs = sizeof(block_q5_0); }; template<> @@ -1049,6 +1070,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q5_1> { static constexpr int qk = QK5_1; static constexpr int qr = QR5_1; static constexpr int qi = QI5_1; + static constexpr int bs = sizeof(block_q5_1); }; template<> @@ -1056,6 +1078,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q8_0> { static constexpr int qk = QK8_0; static constexpr int qr = QR8_0; static constexpr int qi = QI8_0; + static constexpr int bs = sizeof(block_q8_0); }; template<> @@ -1063,6 +1086,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_MXFP4> { static constexpr int qk = QK_MXFP4; static constexpr int qr = QR_MXFP4; static constexpr int qi = QI_MXFP4; + static constexpr int bs = sizeof(block_mxfp4); }; template<> @@ -1070,6 +1094,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_NVFP4> { static constexpr int qk = QK_NVFP4; static constexpr int qr = QR_NVFP4; static constexpr int qi = QI_NVFP4; + static constexpr int bs = sizeof(block_nvfp4); }; template<> @@ -1077,6 +1102,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q2_K> { static constexpr int qk = QK_K; static constexpr int qr = QR2_K; static constexpr int qi = QI2_K; + static constexpr int bs = sizeof(block_q2_K); }; template<> @@ -1084,6 +1110,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q3_K> { static constexpr int qk = QK_K; static constexpr int qr = QR3_K; static constexpr int qi = QI3_K; + static constexpr int bs = sizeof(block_q3_K); }; template<> @@ -1091,6 +1118,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q4_K> { static constexpr int qk = QK_K; static constexpr int qr = QR4_K; static constexpr int qi = QI4_K; + static constexpr int bs = sizeof(block_q4_K); }; template<> @@ -1098,6 +1126,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q5_K> { static constexpr int qk = QK_K; static constexpr int qr = QR5_K; static constexpr int qi = QI5_K; + static constexpr int bs = sizeof(block_q5_K); }; template<> @@ -1105,6 +1134,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q6_K> { static constexpr int qk = QK_K; static constexpr int qr = QR6_K; static constexpr int qi = QI6_K; + static constexpr int bs = sizeof(block_q6_K); }; template<> @@ -1112,6 +1142,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ2_XXS> { static constexpr int qk = QK_K; static constexpr int qr = QR2_XXS; static constexpr int qi = QI2_XXS; + static constexpr int bs = sizeof(block_iq2_xxs); }; template<> @@ -1119,6 +1150,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ2_XS> { static constexpr int qk = QK_K; static constexpr int qr = QR2_XS; static constexpr int qi = QI2_XS; + static constexpr int bs = sizeof(block_iq2_xs); }; template<> @@ -1126,6 +1158,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ2_S> { static constexpr int qk = QK_K; static constexpr int qr = QR2_S; static constexpr int qi = QI2_S; + static constexpr int bs = sizeof(block_iq2_s); }; template<> @@ -1133,6 +1166,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ3_XXS> { static constexpr int qk = QK_K; static constexpr int qr = QR3_XXS; static constexpr int qi = QI3_XXS; + static constexpr int bs = sizeof(block_iq3_xxs); }; template<> @@ -1140,6 +1174,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ1_S> { static constexpr int qk = QK_K; static constexpr int qr = QR1_S; static constexpr int qi = QI1_S; + static constexpr int bs = sizeof(block_iq1_s); }; template<> @@ -1147,6 +1182,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ1_M> { static constexpr int qk = QK_K; static constexpr int qr = QR1_M; static constexpr int qi = QI1_M; + static constexpr int bs = sizeof(block_iq1_m); }; template<> @@ -1154,6 +1190,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ4_NL> { static constexpr int qk = QK4_NL; static constexpr int qr = QR4_NL; static constexpr int qi = QI4_NL; + static constexpr int bs = sizeof(block_iq4_nl); }; template<> @@ -1161,6 +1198,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ4_XS> { static constexpr int qk = QK_K; static constexpr int qr = QR4_XS; static constexpr int qi = QI4_XS; + static constexpr int bs = sizeof(block_iq4_xs); }; template<> @@ -1168,6 +1206,7 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ3_S> { static constexpr int qk = QK_K; static constexpr int qr = QR3_S; static constexpr int qi = QI3_S; + static constexpr int bs = sizeof(block_iq3_s); }; ////////////////////// diff --git a/ggml/src/ggml-cuda/conv2d.cu b/ggml/src/ggml-cuda/conv2d.cu index 14774d4a5e73..10109ad36501 100644 --- a/ggml/src/ggml-cuda/conv2d.cu +++ b/ggml/src/ggml-cuda/conv2d.cu @@ -1,5 +1,6 @@ #include "conv2d.cuh" #include "convert.cuh" +#include "mma.cuh" struct conv_params { const int64_t IW, IH; @@ -111,6 +112,220 @@ static void conv2d_cuda(const float * X_D, const T * K_D, float * Y_D, const con conv2d_kernel<T, whcn_layout><<<blocks, CUDA_CONV2D_BLOCK_SIZE, 0, st>>>(X_D, K_D, Y_D, P); } +static __global__ void +conv2d_pad_f16(const float * input, half * output, int iw, int ih, int pw, int ph, int px, int py, int total) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= total) { + return; + } + const int x = i % pw - px, y = i / pw % ph - py, nc = i / (pw * ph); + output[i] = __float2half( + (unsigned) x < (unsigned) iw && (unsigned) y < (unsigned) ih ? input[(nc * ih + y) * iw + x] : 0.0f); +} + +template <int KW, int KH, bool use_mma> +static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input, + const half * __restrict__ weight, + float * __restrict__ output, + const conv_params P, + const int split_k) { + using namespace ggml_cuda_mma; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nthreads = 4 * warp_size; + constexpr int BM = 64, BN = 64, BK = 64; + constexpr int AS = BK / 2 + 4; + constexpr int BS = BN / 2 + 4; + __shared__ __align__(16) half2 a_s[BM][AS]; + __shared__ __align__(16) half2 b_s[BK][BS]; + + const int tid = threadIdx.y * warp_size + threadIdx.x; + const int iw = int(P.IW), ih = int(P.IH), ow = int(P.OW), oh = int(P.OH); + const int kw = KW ? KW : int(P.KW), kh = KH ? KH : int(P.KH); + const int ic = int(P.IC), oc = int(P.OC); + const int sx = int(P.ST_X), sy = int(P.ST_Y); + const int dx = int(P.DL_X), dy = int(P.DL_Y); + const int n = blockIdx.z / split_k, split = blockIdx.z % split_k; + const int m0 = blockIdx.y * BM, n0 = blockIdx.x * BN; + + const int k_total = ic * kw * kh; + const int load_lane = warp_size == 32 ? threadIdx.x : threadIdx.x % (BN / 2); + const int load_row = threadIdx.y * (warp_size / (BN / 2)) + (warp_size == 32 ? 0 : threadIdx.x / (BN / 2)); + const int spatial = n0 + 2 * load_lane; + const int spatial0 = min(spatial, ow * oh - 1), spatial1 = min(spatial + 1, ow * oh - 1); + const int y0 = spatial0 / ow, x0 = spatial0 % ow; + const int y1 = spatial1 / ow, x1 = spatial1 % ow; + const int pos0 = y0 * sy * iw + x0 * sx, pos1 = y1 * sy * iw + x1 * sx; + + [[maybe_unused]] const int wm = threadIdx.y / 2 * 32, wn = threadIdx.y % 2 * 32; +#if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + using tile_ab = tile<16, 8, half2, get_input_data_layout()>; +# if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + // AMD accumulator fragments transpose the input fragment's row/column mapping. + using tile_c = tile<16, 16, float, DATA_LAYOUT_J_MAJOR>; +# else + using tile_c = tile<16, 16, float>; +# endif + [[maybe_unused]] tile_c c[2][2]; +#else + if constexpr (use_mma) { + NO_DEVICE_CODE; + return; + } +#endif + constexpr int RM = 4, RN = BM * BN / (nthreads * RM); + [[maybe_unused]] const int simt_m = tid / (BN / RN) * RM, simt_n = tid % (BN / RN) * RN; + [[maybe_unused]] float c_simt[RM][RN] = {}; + const int tiles = (k_total + BK - 1) / BK; + const int begin = int(int64_t(tiles) * split / split_k) * BK; + const int end = int(int64_t(tiles) * (split + 1) / split_k) * BK; + for (int k0 = begin; k0 < end; k0 += BK) { + if (k_total % 8 == 0 && uintptr_t(weight) % 16 == 0) { +#pragma unroll + for (int i = tid; i < BM * BK / 8; i += nthreads) { + const int row = i / (BK / 8), col = 8 * (i % (BK / 8)); + const int4 v = m0 + row < oc && k0 + col < k_total ? + ((const int4 *) weight)[((m0 + row) * k_total + k0 + col) / 8] : + make_int4(0, 0, 0, 0); + *(int4 *) &a_s[row][col / 2] = v; + } + } else { +#pragma unroll + for (int i = tid; i < BM * BK / 2; i += nthreads) { + const int row = i / (BK / 2), col = 2 * (i % (BK / 2)); + half lo = __float2half(0.0f), hi = lo; + if (m0 + row < oc && k0 + col < k_total) { + lo = weight[(m0 + row) * k_total + k0 + col]; + if (k0 + col + 1 < k_total) { + hi = weight[(m0 + row) * k_total + k0 + col + 1]; + } + } + a_s[row][col / 2] = __halves2half2(lo, hi); + } + } +#pragma unroll + for (int k = load_row; k < BK; k += nthreads / (BN / 2)) { + const int ki = k0 + k; + const int ci = ki / (kw * kh), ky = ki / kw % kh, kx = ki % kw; + const int offset = ki < k_total ? (n * ic + ci) * ih * iw + ky * dy * iw + kx * dx : 0; + half lo = __float2half(0.0f), hi = lo; + if (ki < k_total && spatial < ow * oh) { + lo = input[offset + pos0]; + } + if (ki < k_total && spatial + 1 < ow * oh) { + hi = input[offset + pos1]; + } + b_s[k][load_lane] = __halves2half2(lo, hi); + } + __syncthreads(); + if constexpr (use_mma) { +#if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) +# pragma unroll + for (int k = 0; k < BK; k += 16) { + tile_ab a[2], b[2]; +# pragma unroll + for (int i = 0; i < 2; ++i) { + load_ldmatrix(a[i], &a_s[wm + 16 * i][k / 2], AS); + load_ldmatrix_trans(b[i], &b_s[k][(wn + 16 * i) / 2], BS); + } +# pragma unroll + for (int i = 0; i < 2; ++i) { +# pragma unroll + for (int j = 0; j < 2; ++j) { + mma(c[i][j], a[i], b[j]); + } + } + } +#endif + } else { +#pragma unroll 4 + for (int k = 0; k < BK; ++k) { + float a[RM], b[RN]; +#pragma unroll + for (int i = 0; i < RM; ++i) { + a[i] = __half2float(((const half *) a_s[simt_m + i])[k]); + } +#pragma unroll + for (int j = 0; j < RN; ++j) { + b[j] = __half2float(((const half *) b_s[k])[simt_n + j]); + } +#pragma unroll + for (int i = 0; i < RM; ++i) { +#pragma unroll + for (int j = 0; j < RN; ++j) { + c_simt[i][j] += a[i] * b[j]; + } + } + } + } + __syncthreads(); + } + if constexpr (use_mma) { +#if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) +# pragma unroll + for (int i = 0; i < 2; ++i) { +# pragma unroll + for (int j = 0; j < 2; ++j) { +# pragma unroll + for (int l = 0; l < c[i][j].ne; ++l) { + const int co = m0 + wm + 16 * i + c[i][j].get_i(l); + const int pos = n0 + wn + 16 * j + c[i][j].get_j(l); + if (co < oc && pos < ow * oh) { + output[(int64_t(blockIdx.z) * oc + co) * ow * oh + pos] = c[i][j].x[l]; + } + } + } + } +#endif + } else { +#pragma unroll + for (int i = 0; i < RM; ++i) { +#pragma unroll + for (int j = 0; j < RN; ++j) { + const int co = m0 + simt_m + i, pos = n0 + simt_n + j; + if (co < oc && pos < ow * oh) { + output[(int64_t(blockIdx.z) * oc + co) * ow * oh + pos] = c_simt[i][j]; + } + } + } + } +} + +static __global__ void conv2d_reduce_split_k(const float * __restrict__ partial, + float * __restrict__ output, + const int total, + const int per_batch, + const int split_k) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= total) { + return; + } + const int n = i / per_batch; + const float * src = partial + int64_t(n) * (split_k - 1) * per_batch + i; + float sum = 0.0f; + for (int k = 0; k < split_k; ++k) { + sum += src[int64_t(k) * per_batch]; + } + output[i] = sum; +} + +template <bool use_mma> +static void conv2d_launch_implicit_gemm(const half * input, + const half * weight, + float * output, + const conv_params & params, + int split_k, + dim3 grid, + dim3 block, + cudaStream_t stream) { + if (params.KW == 3 && params.KH == 3) { + conv2d_implicit_gemm_f16<3, 3, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k); + } else if (params.KW == 1 && params.KH == 1) { + conv2d_implicit_gemm_f16<1, 1, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k); + } else { + conv2d_implicit_gemm_f16<0, 0, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k); + } +} + static void conv2d_cuda_f16(const float * X_D, const half * K_D, float * Y_D, const conv_params P, cudaStream_t st) { conv2d_cuda<half>(X_D, K_D, Y_D, P, st); } @@ -126,6 +341,7 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const float * X_D = (const float *) input->data; float * Y_D = (float *) dst->data; + GGML_ASSERT(input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); GGML_ASSERT(ggml_is_contiguous(input)); GGML_ASSERT(ggml_is_contiguous(kernel)); GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32); @@ -146,19 +362,86 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { // No cwhn GGML_ASSERT(p[6] == false); - const int IW = input->ne[0]; // input_w - const int IH = input->ne[1]; // input_h - const int OW = dst->ne[0]; // output_w - const int OH = dst->ne[1]; // output_h - const int KW = kernel->ne[0]; // kernel_w - const int KH = kernel->ne[1]; // kernel_h - const int IC = input->ne[2]; // input_channels - const int OC = kernel->ne[3]; // ouptut_chanles - const int B = input->ne[3]; // n_batches + const int64_t IW = input->ne[0]; // input_w + const int64_t IH = input->ne[1]; // input_h + const int64_t OW = dst->ne[0]; // output_w + const int64_t OH = dst->ne[1]; // output_h + const int64_t KW = kernel->ne[0]; // kernel_w + const int64_t KH = kernel->ne[1]; // kernel_h + const int64_t IC = input->ne[2]; // input_channels + const int64_t OC = kernel->ne[3]; // ouptut_chanles + const int64_t B = input->ne[3]; // n_batches const int64_t total = B * OC * OH * OW; conv_params params = { IW, IH, OW, OH, KW, KH, ST_X, ST_Y, PD_X, PD_Y, DL_X, DL_Y, IC, OC, B, total }; + const auto & device = ggml_cuda_info().devices[ctx.device]; + const bool use_mma = + turing_mma_available(device.cc) || amd_wmma_available(device.cc) || amd_mfma_available(device.cc); + // MUSA can share the tiling without a native fragment implementation in mma.cuh. + const bool use_simt = GGML_CUDA_CC_IS_MTHREADS(device.cc); + const bool pointwise = KW == 1 && KH == 1 && ST_X == 1 && ST_Y == 1 && PD_X == 0 && PD_Y == 0; + const bool use_blas = pointwise && fast_fp16_hardware_available(device.cc); + // Short reductions on small maps do not amortize conversion and launch costs. + const bool small_conv = IC * KW * KH < 64 && OW * OH < 512; + + const int64_t limit = INT_MAX - 256; + const int64_t padded_w = IW + 2 * int64_t(PD_X), padded_h = IH + 2 * int64_t(PD_Y); + const bool padded_fits = padded_w > 0 && padded_w <= limit && padded_h > 0 && padded_h <= limit && + padded_w * padded_h <= limit && IC * B <= limit / (padded_w * padded_h); + if (kernel->type == GGML_TYPE_F16 && (use_mma || use_blas || use_simt) && (use_blas || !small_conv) && + ggml_nelements(input) <= limit && ggml_nelements(kernel) <= limit && total <= limit && padded_fits && + PD_X >= 0 && PD_Y >= 0 && ST_X > 0 && ST_Y > 0 && DL_X > 0 && DL_Y > 0 && + (OW - 1) * ST_X + (KW - 1) * DL_X < padded_w && (OH - 1) * ST_Y + (KH - 1) * DL_Y < padded_h && + (OC + 63) / 64 <= 65535 && B <= 65535) { + const int pw = int(padded_w), ph = int(padded_h); + const int padded_total = int(padded_w * padded_h * IC * B); + + ggml_cuda_pool_alloc<half> x_half(ctx.pool(), padded_total); + // Match im2col's F16 input precision, but expand patches only in shared memory and accumulate in F32. + if (PD_X == 0 && PD_Y == 0) { + ggml_get_to_fp16_cuda(input->type)(X_D, x_half.get(), padded_total, st); + } else { + conv2d_pad_f16<<<(padded_total + 255) / 256, 256, 0, st>>>(X_D, x_half.get(), int(IW), int(IH), pw, ph, + PD_X, PD_Y, padded_total); + } + const conv_params padded_params = { pw, ph, OW, OH, KW, KH, ST_X, ST_Y, 0, 0, DL_X, DL_Y, IC, OC, B, total }; + if (use_blas) { + const float alpha = 1.0f, beta = 0.0f; + const int positions = int(OW * OH); + cublasHandle_t cublas_h = ctx.cublas_handle(); + for (int n = 0; n < B; ++n) { + CUBLAS_CHECK(cublasGemmEx(cublas_h, CUBLAS_OP_N, CUBLAS_OP_N, positions, int(OC), int(IC), &alpha, + x_half.get() + int64_t(n) * IC * positions, CUDA_R_16F, positions, K_D, + CUDA_R_16F, int(IC), &beta, Y_D + int64_t(n) * OC * positions, CUDA_R_32F, + positions, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + } + return; + } + const int64_t blocks = ((OW * OH + 63) / 64) * ((OC + 63) / 64) * B; + const int target = 8 * ggml_cuda_info().devices[ctx.device].nsm; + // Split long reductions so small spatial maps still occupy the GPU. + const int split_k = int(std::min({ int64_t(32), int64_t(65535) / B, (IC * KW * KH + 63) / 64, + std::max(int64_t(1), (target + blocks - 1) / blocks) })); + + ggml_cuda_pool_alloc<float> partial(ctx.pool()); + float * result = split_k == 1 ? Y_D : partial.alloc(total * split_k); + const dim3 block(device.warp_size, 4); + const dim3 grid(unsigned((OW * OH + 63) / 64), unsigned((OC + 63) / 64), unsigned(B * split_k)); + if (use_mma) { + conv2d_launch_implicit_gemm<true>(x_half.get(), (const half *) K_D, result, padded_params, split_k, grid, + block, st); + } else { + conv2d_launch_implicit_gemm<false>(x_half.get(), (const half *) K_D, result, padded_params, split_k, grid, + block, st); + } + if (split_k > 1) { + conv2d_reduce_split_k<<<(total + 255) / 256, 256, 0, st>>>(result, Y_D, int(total), int(OC * OW * OH), + split_k); + } + return; + } + if (kernel->type == GGML_TYPE_F16) { conv2d_cuda_f16(X_D, (half *) K_D, Y_D, params, st); } else { diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index ae8f5b2fc02c..5ad8577e6f1a 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -440,6 +440,29 @@ static __global__ void convert_unary( } } +template <typename T> struct alignas(sizeof(T)*4) cvt_vec4 { T v[4]; }; + +// four elements per thread, so a warp moves 512B (RDNA) / 1k (CDNA) per load +template <typename src_t, typename dst_t> +static __global__ void convert_unary_cont_vec4( + const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k4) { + const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x; + + if (i >= k4) { + return; + } + + const cvt_vec4<src_t> xv = ((const cvt_vec4<src_t> *) vx)[i]; + + cvt_vec4<dst_t> yv; +#pragma unroll + for (int j = 0; j < 4; ++j) { + yv.v[j] = ggml_cuda_cast<dst_t>(xv.v[j]); + } + + ((cvt_vec4<dst_t> *) y)[i] = yv; +} + template <typename src_t, typename dst_t> static void convert_unary_cuda(const void * vx, dst_t * y, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03, @@ -453,6 +476,15 @@ static void convert_unary_cuda(const void * vx, dst_t * y, template <typename src_t, typename dst_t> static void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + if (k % 4 == 0 && + (uintptr_t) vx % alignof(cvt_vec4<src_t>) == 0 && + (uintptr_t) y % alignof(cvt_vec4<dst_t>) == 0) { + const int64_t k4 = k/4; + const int64_t num_blocks = (k4 + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE; + convert_unary_cont_vec4<src_t, dst_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k4); + return; + } + convert_unary_cuda<src_t>(vx, y, k, 1, 1, 1, k, k, k, stream); } diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu index 5ff380a6d999..5bd2712757d4 100644 --- a/ggml/src/ggml-cuda/cpy.cu +++ b/ggml/src/ggml-cuda/cpy.cu @@ -752,6 +752,14 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg ggml_cpy_scalar_cuda<int32_t, int32_t> (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } + } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16) { + if (can_be_transposed) { + ggml_cpy_scalar_cuda<int16_t, int16_t, true> + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else { + ggml_cpy_scalar_cuda<int16_t, int16_t> + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_I32) { if (contiguous_srcs) { ggml_cpy_scalar_contiguous_cuda<float, int32_t> diff --git a/ggml/src/ggml-cuda/dsv4-hc.cu b/ggml/src/ggml-cuda/dsv4-hc.cu index c4b19a787b0e..ca1d2dc8a482 100644 --- a/ggml/src/ggml-cuda/dsv4-hc.cu +++ b/ggml/src/ggml-cuda/dsv4-hc.cu @@ -100,6 +100,7 @@ static __global__ void dsv4_hc_comb_f32( } } +template <bool gated> static __global__ void dsv4_hc_pre_f32( const float * x, const float * weights, @@ -112,8 +113,10 @@ static __global__ void dsv4_hc_pre_f32( int64_t sx2, int64_t sw0, int64_t sw1, + int64_t sw2, int64_t sd0, - int64_t sd1) { + int64_t sd1, + float scale) { ggml_cuda_pdl_lc(); const int64_t ir = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; const int64_t nr = n_embd * n_tokens; @@ -127,16 +130,22 @@ static __global__ void dsv4_hc_pre_f32( const int64_t i0 = ir % n_embd; const int64_t it = ir / n_embd; - float sum = x[i0*sx0 + it*sx2] * weights[it*sw1]; - for (int64_t ih = 1; ih < hc; ++ih) { + float sum = 0.0f; + for (int64_t ih = 0; ih < hc; ++ih) { const float xv = x[i0*sx0 + ih*sx1 + it*sx2]; - const float wv = weights[ih*sw0 + it*sw1]; + float wv; + if constexpr (gated) { + wv = 1.0f / (1.0f + expf(-weights[i0*sw0 + ih*sw1 + it*sw2])); + } else { + wv = weights[ih*sw0 + it*sw1]; + } sum += xv * wv; } - dst[i0*sd0 + it*sd1] = sum; + dst[i0*sd0 + it*sd1] = scale * sum; } +template <bool has_comb> static __global__ void dsv4_hc_post_f32( const float * x, const float * residual, @@ -174,8 +183,12 @@ static __global__ void dsv4_hc_post_f32( const int64_t it = ir / (n_embd * hc); float sum = x[i0*sx0 + it*sx1] * post[idst*sp0 + it*sp1]; - for (int64_t isrc = 0; isrc < hc; ++isrc) { - sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2]; + if constexpr (has_comb) { + for (int64_t isrc = 0; isrc < hc; ++isrc) { + sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2]; + } + } else { + sum += residual[i0*sr0 + idst*sr1 + it*sr2]; } dst[i0*sd0 + idst*sd1 + it*sd2] = sum; @@ -240,18 +253,23 @@ void ggml_cuda_op_dsv4_hc_pre(ggml_backend_cuda_context & ctx, ggml_tensor * dst const int64_t hc = x->ne[1]; const int64_t n_tokens = x->ne[2]; + const float scale = ggml_get_op_params_f32(dst, 0); + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + const int block_size = 256; const int64_t nr = n_embd * n_tokens; const dim3 block_dims(block_size, 1, 1); const dim3 grid_dims((nr + block_size - 1) / block_size, 1, 1); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream()); - ggml_cuda_kernel_launch(dsv4_hc_pre_f32, launch_params, + auto kernel = gated ? dsv4_hc_pre_f32<true> : dsv4_hc_pre_f32<false>; + ggml_cuda_kernel_launch(kernel, launch_params, (const float *) x->data, (const float *) weights->data, (float *) dst->data, n_embd, hc, n_tokens, nbx0 / sizeof(float), nbx1 / sizeof(float), nbx2 / sizeof(float), - nbw0 / sizeof(float), nbw1 / sizeof(float), - nbd0 / sizeof(float), nbd1 / sizeof(float)); + nbw0 / sizeof(float), nbw1 / sizeof(float), nbw2 / sizeof(float), + nbd0 / sizeof(float), nbd1 / sizeof(float), + scale); } void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { @@ -263,15 +281,18 @@ void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * ds GGML_ASSERT(x->type == GGML_TYPE_F32); GGML_ASSERT(residual->type == GGML_TYPE_F32); GGML_ASSERT(post->type == GGML_TYPE_F32); - GGML_ASSERT(comb->type == GGML_TYPE_F32); + GGML_ASSERT(comb == nullptr || comb->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); GGML_TENSOR_LOCALS(size_t, nbx, x, nb); GGML_TENSOR_LOCALS(size_t, nbr, residual, nb); GGML_TENSOR_LOCALS(size_t, nbp, post, nb); - GGML_TENSOR_LOCALS(size_t, nbc, comb, nb); GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + const size_t nbc0 = comb ? comb->nb[0] : 0; + const size_t nbc1 = comb ? comb->nb[1] : 0; + const size_t nbc2 = comb ? comb->nb[2] : 0; + const int64_t n_embd = x->ne[0]; const int64_t n_tokens = x->ne[1]; const int64_t hc = residual->ne[1]; @@ -282,9 +303,10 @@ void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * ds const dim3 grid_dims((nr + block_size - 1) / block_size, 1, 1); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream()); - ggml_cuda_kernel_launch(dsv4_hc_post_f32, launch_params, + auto kernel = comb ? dsv4_hc_post_f32<true> : dsv4_hc_post_f32<false>; + ggml_cuda_kernel_launch(kernel, launch_params, (const float *) x->data, (const float *) residual->data, - (const float *) post->data, (const float *) comb->data, (float *) dst->data, + (const float *) post->data, comb ? (const float *) comb->data : nullptr, (float *) dst->data, n_embd, hc, n_tokens, nbx0 / sizeof(float), nbx1 / sizeof(float), nbr0 / sizeof(float), nbr1 / sizeof(float), nbr2 / sizeof(float), diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index d68783b124a7..683fa4ba2b89 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -1108,7 +1108,7 @@ static __global__ void flash_attn_mask_to_KV_max( } void ggml_cuda_flash_attn_ext_compact_mask( - const ggml_tensor * mask, int32_t * indices, int32_t n_kv_max, cudaStream_t stream); + const ggml_tensor * mask, int32_t * indices, int32_t * counts, int32_t n_queries, int32_t ncols1, int32_t n_kv_max, cudaStream_t stream); template<int D, int ncols1, int ncols2> // D == head size __launch_bounds__(D, 1) @@ -1481,14 +1481,18 @@ void launch_fattn( const int ntiles_z_gqa = ((gqa_ratio + ncols2 - 1) / ncols2); const int ntiles_dst = ntiles_x * ntiles_z_gqa * K->ne[2] * Q->ne[3]; - const int32_t n_kv_max = use_sparse ? ggml_get_op_params_i32(KQV, 4) : 0; + // sparse: a query tile of ncols1 queries shares one index list, the union of the queries' visible columns + int32_t n_kv_max = 0; if (use_sparse) { GGML_ASSERT(mask != nullptr); - GGML_ASSERT(n_kv_max > 0); - const size_t mask_rows = size_t(mask->ne[1]) * mask->ne[3]; + const int32_t n_kv_max_query = ggml_get_op_params_i32(KQV, 4); + GGML_ASSERT(n_kv_max_query > 0); + n_kv_max = std::min<int64_t>(K->ne[1], int64_t(ncols1)*n_kv_max_query); - KV_max.alloc(size_t(n_kv_max) * mask_rows); - ggml_cuda_flash_attn_ext_compact_mask(mask, KV_max.ptr, n_kv_max, main_stream); + const size_t n_lists = size_t(ntiles_x) * mask->ne[3]; + + KV_max.alloc(size_t(n_kv_max)*n_lists + n_lists); + ggml_cuda_flash_attn_ext_compact_mask(mask, KV_max.ptr, KV_max.ptr + size_t(n_kv_max)*n_lists, Q->ne[1], ncols1, n_kv_max, main_stream); } // Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped. @@ -1522,12 +1526,21 @@ void launch_fattn( dim3 blocks_num; if (stream_k) { - // For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup. - const int max_blocks = max_blocks_per_sm*nsm; - const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; - const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves); + auto should_use_stream_k = [](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) { + const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; + const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves); + + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_ADA_LOVELACE) { + return true; + } + if (amd_wmma_available(cc) && DKQ == 64) { + return true; // TODO better configuration + } + return tiles_efficiency_percent < 75; + }; - const bool use_stream_k = cc >= GGML_CUDA_CC_ADA_LOVELACE || amd_wmma_available(cc) || tiles_efficiency_percent < 75; + const int max_blocks = max_blocks_per_sm*nsm; + const bool use_stream_k = should_use_stream_k(cc, ntiles_dst, max_blocks, Q->ne[0]); blocks_num.x = ntiles_dst; blocks_num.y = 1; @@ -1611,8 +1624,8 @@ void launch_fattn( GGML_ASSERT(block_dim.x % warp_size == 0); - ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num, block_dim, nbytes_shared, main_stream); - ggml_cuda_kernel_launch(fattn_kernel, launch_params, + ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num, block_dim, nbytes_shared, main_stream); + ggml_cuda_kernel_launch(fattn_kernel, launch_params, (const char *) Q->data, K_data, V_data, diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index b2518c0c25f6..f347d7857b90 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -2,7 +2,6 @@ #include "cp-async.cuh" #include "mma.cuh" #include "fattn-common.cuh" -#include "fattn-swizzle.cuh" using namespace ggml_cuda_mma; @@ -68,16 +67,16 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 64, 128, 2, 32, 96, 64, 64, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 128, 2, 64, 128, 128, 128, 2, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 4, 32, 128, 128, 128, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 256, 1, 64, 128, 128, 128, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 32, 128, 128, 128, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 32, 128, 128, 128, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2, 32, 128, 128, 128, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 256, 1, 32, 128, 128, 128, 1, false); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 8, 64, 4, 32, 256, 256, 128, 1, false); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16, 64, 4, 32, 256, 256, 128, 1, false); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 128, 2, 32, 128, 128, 128, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 8, 128, 2, 64, 128, 128, 128, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16, 256, 1, 64, 128, 128, 128, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 256, 1, 32, 128, 128, 128, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 64, 256, 1, 32, 128, 128, 128, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 8, 64, 4, 32, 288, 256, 128, 1, false); @@ -173,8 +172,8 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 2, 32, 128, 128, 128, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 2, 32, 128, 128, 128, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 64, 128, 128, 64, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 64, 128, 128, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 256, 2, 64, 128, 128, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 2, 64, 128, 128, 64, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2, 32, 160, 128, 128, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 128, 2, 32, 160, 128, 128, 1, true); @@ -200,7 +199,7 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 1, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 256, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 32, 32, 32, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 4, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 3, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 256, 2, 64, 40, 40, 40, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 256, 2, 64, 40, 40, 40, 1, true); @@ -372,6 +371,32 @@ static constexpr __device__ bool ggml_cuda_fattn_mma_get_Q_in_reg(const int DKQ, return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).Q_in_reg; } +// Swizzling needs a tile stride that is a multiple of 32 half2 columns. +static constexpr __host__ __device__ bool ggml_cuda_fattn_mma_bank_aligned(const int nbatch_2) { + return nbatch_2 >= 32 && nbatch_2 % 32 == 0; +} + +// Swizzling needs ldmatrix, on other hardware the tiles keep the row padding. +static __host__ bool ggml_cuda_fattn_mma_get_swizzled(const int DKQ, const int DV, const int ncols1, const int ncols2, const int cc) { + const fattn_mma_config cfg = ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols1*ncols2, cc); + return turing_mma_available(cc) && ggml_cuda_fattn_mma_bank_aligned(cfg.nbatch_K2) && ggml_cuda_fattn_mma_bank_aligned(cfg.nbatch_V2); +} + +static constexpr __device__ bool ggml_cuda_fattn_mma_get_swizzled(const int DKQ, const int DV, const int ncols1, const int ncols2) { +#if defined(TURING_MMA_AVAILABLE) + const fattn_mma_config cfg = ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols1*ncols2); + return ggml_cuda_fattn_mma_bank_aligned(cfg.nbatch_K2) && ggml_cuda_fattn_mma_bank_aligned(cfg.nbatch_V2); +#else + GGML_UNUSED_VARS(DKQ, DV, ncols1, ncols2); + return false; +#endif // defined(TURING_MMA_AVAILABLE) +} + +// Row padding is only needed if the tile is not swizzled. +static constexpr __host__ __device__ int ggml_cuda_fattn_mma_get_stride_tile(const int nbatch_2, const bool swizzled) { + return swizzled ? nbatch_2 : nbatch_2 + 4; +} + static constexpr __device__ int get_cols_per_thread() { #if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) return 1; // AMD has a single column per thread. @@ -456,12 +481,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k); - if constexpr (swz) { - const int smem_offs_b = ggml_cuda_fattn_smem_swizzle::bytes_rc<stride_tile>(i, k*h2_per_chunk); - cp_async_cg_16<preload>(tile_KV_32 + smem_offs_b, KV + i_KV*stride_KV + k*h2_per_chunk); - } else { - cp_async_cg_16<preload>(tile_KV_32 + i*(stride_tile*sizeof(half2)) + k*16, KV + i_KV*stride_KV + k*h2_per_chunk); - } + cp_async_cg_16<preload>(tile_KV_32 + swizzle_bytes<swz, half2>(i, k*h2_per_chunk, stride_tile), KV + i_KV*stride_KV + k*h2_per_chunk); } } }; @@ -503,11 +523,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( } else { src = !oob_check || i < i_sup ? KV + int64_t(k_VKQ_0 + i)*stride_KV + k*h2_per_chunk : zero; } - if constexpr (swz) { - ggml_cuda_memcpy_1<16>((char *) tile_KV + ggml_cuda_fattn_smem_swizzle::bytes_rc<stride_tile>(i, k*h2_per_chunk), src); - } else { - ggml_cuda_memcpy_1<16>(tile_KV + i*stride_tile + k*4, src); - } + ggml_cuda_memcpy_1<16>((char *) tile_KV + swizzle_bytes<swz, half2>(i, k*h2_per_chunk, stride_tile), src); } } }; @@ -650,11 +666,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, use_sparse); - // swizzle the tile stride for K and V based on the batch size. - constexpr int stride_tile_K = ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_K2); - constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_V2); - constexpr bool swz_K = ggml_cuda_fattn_smem_swizzle::enabled(nbatch_K2); - constexpr bool swz_V = V_is_K_view ? swz_K : ggml_cuda_fattn_smem_swizzle::enabled(nbatch_V2); + constexpr bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols1, ncols2); + constexpr int stride_tile_K = ggml_cuda_fattn_mma_get_stride_tile(nbatch_K2, swz); + constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : ggml_cuda_fattn_mma_get_stride_tile(nbatch_V2, swz); const int k_VKQ_0 = kb0 * nbatch_fa; #if defined(TURING_MMA_AVAILABLE) @@ -672,7 +686,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( constexpr bool use_cp_async = true; cp_async_wait_all(); __syncthreads(); - flash_attn_ext_f16_load_tile<stride_tile_V, swz_V, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> + flash_attn_ext_f16_load_tile<stride_tile_V, swz, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> (V_h2, tile_V, nbatch_V2, stride_V, k_VKQ_0, k_VKQ_sup, nullptr); } else { // the sparse mask values are gathered per element, always load them synchronously @@ -692,7 +706,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( if constexpr (nstages <= 1) { const int k0_diff = k0_stop - k0_start; constexpr bool use_cp_async = nstages == 1; - flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> + flash_attn_ext_f16_load_tile<stride_tile_K, swz, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> (K_h2 + k0_start, tile_K, k0_diff, stride_K, k_VKQ_0, k_VKQ_sup, indices); if (use_cp_async) { cp_async_wait_all(); @@ -708,7 +722,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( #pragma unroll for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) { T_A_KQ K_A; - ggml_cuda_fattn_smem_swizzle::load_ldmatrix<stride_tile_K, swz_K>(K_A, tile_K, i_KQ_0, k_KQ_0 - k0_start); + load_ldmatrix<swz>(K_A, tile_K, i_KQ_0, k_KQ_0 - k0_start, stride_tile_K); if constexpr (cols_per_warp == 8) { mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[k_KQ_0/T_A_KQ::J]); } else { @@ -734,7 +748,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( const int i_KQ_0 = i_KQ_00 + (threadIdx.y % np)*T_A_KQ::I; T_A_KQ K_A; - ggml_cuda_fattn_smem_swizzle::load_ldmatrix<stride_tile_K, swz_K>(K_A, tile_K, i_KQ_0, k_KQ_0 - k0_start); + load_ldmatrix<swz>(K_A, tile_K, i_KQ_0, k_KQ_0 - k0_start, stride_tile_K); if constexpr (cols_per_warp == 8) { mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[0]); @@ -1029,7 +1043,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> (mask_h, tile_mask, stride_mask, k_VKQ_0 + nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr); } - flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> + flash_attn_ext_f16_load_tile<stride_tile_K, swz, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> (K_h2, tile_K, nbatch_K2, stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, nullptr); } } @@ -1045,7 +1059,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( const int i0_diff = i0_stop - i0_start; if (!V_is_K_view || i0_stop > 2*nbatch_K2) { constexpr bool use_cp_async = nstages == 1; - flash_attn_ext_f16_load_tile<stride_tile_V, swz_V, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> + flash_attn_ext_f16_load_tile<stride_tile_V, swz, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> (V_h2 + i0_start/2, tile_V, i0_diff/2, stride_V, k_VKQ_0, k_VKQ_sup, indices); if (use_cp_async) { cp_async_wait_all(); @@ -1064,7 +1078,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( const int k0 = k00 + (threadIdx.y % np)*T_A_VKQ::J; T_A_VKQ A; // Transposed in SRAM but not in registers, gets transposed on load. - ggml_cuda_fattn_smem_swizzle::load_ldmatrix_trans<stride_tile_V, swz_V>(A, tile_V, (int)(tile_V_i - tile_V) + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2); + load_ldmatrix_trans<swz>(A, tile_V, 2*k0, (int)(tile_V_i - tile_V) + (i_VKQ_0 - i0_start)/2, stride_tile_V); if constexpr (T_B_KQ::I == 8) { mma(VKQ_C[i_VKQ_0/T_A_VKQ::I], A, B[k00/(np*T_A_VKQ::J)]); } else { @@ -1090,7 +1104,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( const int k0 = k00 + (threadIdx.y % np)*T_A_VKQ::I; T_A_VKQ A; // Transposed in both SRAM and registers, load normally. - ggml_cuda_fattn_smem_swizzle::load_ldmatrix<stride_tile_V, swz_V>(A, tile_V, (int)(tile_V_i - tile_V) + k0*stride_tile_V + (i_VKQ_0 - i0_start)/2); + static_assert(!swz, "Volta has no ldmatrix"); + load_ldmatrix(A, tile_V_i + k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V); mma(VKQ_C[i_VKQ_0/i0_stride], B[k00/(np*T_A_VKQ::I)], A); } } @@ -1111,7 +1126,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( } #if defined(TURING_MMA_AVAILABLE) -template<int DV, int ncols> struct mma_tile_sizes { +template<int DKQ, int ncols> struct mma_tile_sizes { using T_A_KQ = tile<16, 8, half2>; // row-major using T_B_KQ = tile<16, 8, half2>; // column-major using T_C_KQ = tile<16, 16, float>; // column-major @@ -1119,7 +1134,33 @@ template<int DV, int ncols> struct mma_tile_sizes { using T_B_VKQ = tile<16, 8, half2>; // column-major using T_C_VKQ = tile<16, 8, half2>; // column-major }; -template<int DV> struct mma_tile_sizes<DV, 8> { +// If there are only 8 columns, use thinner B tiles to avoid wasting compute: +template<int DKQ> struct mma_tile_sizes<DKQ, 8> { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile< 8, 8, half2>; // column-major + using T_C_KQ = tile<16, 8, float>; // row-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile< 8, 8, half2>; // column-major + using T_C_VKQ = tile<16, 4, half2>; // row-major +}; +// For very large head sizes, use thinner B tiles to reduce register pressure: +template<> struct mma_tile_sizes<256, 16> { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile< 8, 8, half2>; // column-major + using T_C_KQ = tile<16, 8, float>; // row-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile< 8, 8, half2>; // column-major + using T_C_VKQ = tile<16, 4, half2>; // row-major +}; +template<> struct mma_tile_sizes<512, 16> { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile< 8, 8, half2>; // column-major + using T_C_KQ = tile<16, 8, float>; // row-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile< 8, 8, half2>; // column-major + using T_C_VKQ = tile<16, 4, half2>; // row-major +}; +template<> struct mma_tile_sizes<512, 32> { using T_A_KQ = tile<16, 8, half2>; // row-major using T_B_KQ = tile< 8, 8, half2>; // column-major using T_C_KQ = tile<16, 8, float>; // row-major @@ -1129,7 +1170,7 @@ template<int DV> struct mma_tile_sizes<DV, 8> { }; #elif defined(AMD_WMMA_AVAILABLE) #ifdef RDNA3 -template<int DV, int ncols> struct mma_tile_sizes { +template<int DKQ, int ncols> struct mma_tile_sizes { using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major using T_B_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major using T_C_KQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major @@ -1154,7 +1195,7 @@ template<int ncols> struct mma_tile_sizes<112, ncols> { using T_C_VKQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major }; #else -template<int DV, int ncols> struct mma_tile_sizes { +template<int DKQ, int ncols> struct mma_tile_sizes { using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR>; // row-major using T_B_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR>; // column-major using T_C_KQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major @@ -1180,16 +1221,16 @@ template<int ncols> struct mma_tile_sizes<112, ncols> { }; #endif // RDNA3 #elif defined(AMD_MFMA_AVAILABLE) -template<int DV, int ncols> struct mma_tile_sizes { +template<int DKQ, int ncols> struct mma_tile_sizes { using T_A_KQ = tile<16, 8, half2>; // row-major using T_B_KQ = tile<16, 8, half2>; // column-major using T_C_KQ = tile<16, 16, float>; // column-major using T_A_VKQ = tile<16, 8, half2>; // row-major using T_B_VKQ = tile<16, 8, half2>; // column-major - using T_C_VKQ = tile<16, 8, half2>; // column-major + using T_C_VKQ = tile<16, 16, float>; // column-major }; #else // Volta -template<int DV, int ncols> struct mma_tile_sizes { +template<int DKQ, int ncols> struct mma_tile_sizes { using T_A_KQ = tile< 8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major using T_B_KQ = tile<32, 4, half2, DATA_LAYOUT_I_MAJOR>; // column-major using T_C_KQ = tile<32, 8, float, DATA_LAYOUT_I_MAJOR>; // column-major @@ -1230,12 +1271,12 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int ncols = ncols1 * ncols2; - using T_A_KQ = typename mma_tile_sizes<DV, ncols>::T_A_KQ; - using T_B_KQ = typename mma_tile_sizes<DV, ncols>::T_B_KQ; - using T_C_KQ = typename mma_tile_sizes<DV, ncols>::T_C_KQ; - using T_A_VKQ = typename mma_tile_sizes<DV, ncols>::T_A_VKQ; - using T_B_VKQ = typename mma_tile_sizes<DV, ncols>::T_B_VKQ; - using T_C_VKQ = typename mma_tile_sizes<DV, ncols>::T_C_VKQ; + using T_A_KQ = typename mma_tile_sizes<DKQ, ncols>::T_A_KQ; + using T_B_KQ = typename mma_tile_sizes<DKQ, ncols>::T_B_KQ; + using T_C_KQ = typename mma_tile_sizes<DKQ, ncols>::T_C_KQ; + using T_A_VKQ = typename mma_tile_sizes<DKQ, ncols>::T_A_VKQ; + using T_B_VKQ = typename mma_tile_sizes<DKQ, ncols>::T_B_VKQ; + using T_C_VKQ = typename mma_tile_sizes<DKQ, ncols>::T_C_VKQ; constexpr int cols_per_warp = T_B_KQ::I; constexpr int cols_per_thread = get_cols_per_thread(); @@ -1255,12 +1296,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( static_assert(nwarps * (cols_per_warp/ncols2) % ncols1 == 0, "bad nwarps"); constexpr int stride_tile_Q = DKQ/2 + 4; - // swizzle the tile stride for K and V based on the batch size. - constexpr int stride_tile_K = ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_K2); - constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_V2); + constexpr bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols1, ncols2); + constexpr int stride_tile_K = ggml_cuda_fattn_mma_get_stride_tile(nbatch_K2, swz); + constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : ggml_cuda_fattn_mma_get_stride_tile(nbatch_V2, swz); constexpr int stride_tile_KV_max = stride_tile_K > stride_tile_V ? stride_tile_K : stride_tile_V; - constexpr bool swz_K = ggml_cuda_fattn_smem_swizzle::enabled(nbatch_K2); - constexpr bool swz_V = V_is_K_view ? swz_K : ggml_cuda_fattn_smem_swizzle::enabled(nbatch_V2); extern __shared__ half2 tile_Q[]; half2 * tile_K = Q_in_reg ? tile_Q : tile_Q + ncols * stride_tile_Q; @@ -1272,7 +1311,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)]; #elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3) T_C_VKQ VKQ_C[DV % 32 != 0 ? DV/T_C_VKQ::J : DV/(2*T_C_VKQ::J)]; -#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) +#elif defined(AMD_MFMA_AVAILABLE) + T_C_VKQ VKQ_C[ DV/T_C_VKQ::J]; +#elif defined(AMD_WMMA_AVAILABLE) T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)]; #else // Volta T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)]; @@ -1355,7 +1396,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> (mask_h, tile_mask, stride_mask, kb0*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr); } - flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> + flash_attn_ext_f16_load_tile<stride_tile_K, swz, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse> (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, nullptr); } @@ -1520,14 +1561,12 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( constexpr int tile_stride = nbatch_combine + 4; static_assert((DV/2) % nbatch_combine == 0, "bad nbatch_combine"); - constexpr bool combine_needs_sync = swz_K || swz_V; - if constexpr (cols_per_warp == 8) { const int jc_cwmo = (threadIdx.x % (2*T_C_VKQ::J)) / T_C_VKQ::J; // jc combine write meta offset const int jc_cwm = threadIdx.y*(2*T_C_VKQ::J) + 2*T_C_VKQ::get_j(-1) + jc_cwmo; // jc combine write meta const float2 KQ_cmr = make_float2(KQ_max[jc_cwmo], KQ_rowsum[jc_cwmo]); // KQ combine max rowsum - if constexpr (combine_needs_sync) { + if constexpr (swz) { __syncthreads(); } @@ -1567,7 +1606,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const bool thread_should_write = T_C_KQ::J == 8 || T_C_KQ::get_j(threadIdx.x & 2) < 8; #endif // defined(TURING_MMA_AVAILABLE) - if constexpr (combine_needs_sync) { + if constexpr (swz) { __syncthreads(); } @@ -1590,77 +1629,77 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } } - if (np > 1 && threadIdx.y % np == 0) { - // Combine the meta data for parallel warps via shared memory. - // Warps with threadIdx.y % np != 0 must NOT return early. - // All threads must return simultaneously to avoid race conditions with work on the next tile. - + if (np > 1) { constexpr int nmeta = np*cols_per_warp >= warp_size ? np*cols_per_warp/warp_size : 1; + float KQ_cmn; + float KQ_cms[nmeta]; + float KQ_crs; + const int jc_meta = threadIdx.y*cols_per_warp + (np*cols_per_warp < warp_size ? threadIdx.x % (np*cols_per_warp) : threadIdx.x); float2 * const meta_ptr = ((float2 *) tile_Q) + jc_meta*(tile_stride/2) + nbatch_combine/2; - float2 meta[nmeta]; + + if (threadIdx.y % np == 0) { + // Combine the meta data for parallel warps via shared memory. + float2 meta[nmeta]; #pragma unroll - for (int imeta = 0; imeta < nmeta; ++imeta) { - meta[imeta] = meta_ptr[imeta * warp_size * tile_stride/2]; - } + for (int imeta = 0; imeta < nmeta; ++imeta) { + meta[imeta] = meta_ptr[imeta * warp_size * tile_stride/2]; + } - float KQ_cmn = meta[0].x; // KQ combine max new, max between all parallel warps. + KQ_cmn = meta[0].x; // KQ combine max new, max between all parallel warps. #pragma unroll - for (int imeta = 1; imeta < nmeta; ++imeta) { - KQ_cmn = fmaxf(KQ_cmn, meta[imeta].x); - } + for (int imeta = 1; imeta < nmeta; ++imeta) { + KQ_cmn = fmaxf(KQ_cmn, meta[imeta].x); + } #pragma unroll - for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) { - if (offset < warp_size) { - KQ_cmn = fmaxf(KQ_cmn, __shfl_xor_sync(0xFFFFFFFF, KQ_cmn, offset, warp_size)); + for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) { + if (offset < warp_size) { + KQ_cmn = fmaxf(KQ_cmn, __shfl_xor_sync(0xFFFFFFFF, KQ_cmn, offset, warp_size)); + } } - } - float KQ_cms[nmeta]; // KQ combine max scale per warp. #pragma unroll - for (int imeta = 0; imeta < nmeta; ++imeta) { - KQ_cms[imeta] = expf(meta[imeta].x - KQ_cmn); - } + for (int imeta = 0; imeta < nmeta; ++imeta) { + KQ_cms[imeta] = expf(meta[imeta].x - KQ_cmn); + } - float KQ_crs = KQ_cms[0]*meta[0].y; // KQ combine rowsum, scaled sum of all parallel warps. + KQ_crs = KQ_cms[0]*meta[0].y; // KQ combine rowsum, scaled sum of all parallel warps. #pragma unroll - for (int imeta = 1; imeta < nmeta; ++imeta) { - KQ_crs += KQ_cms[imeta]*meta[imeta].y; - } + for (int imeta = 1; imeta < nmeta; ++imeta) { + KQ_crs += KQ_cms[imeta]*meta[imeta].y; + } #pragma unroll - for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) { - if (offset < warp_size) { - KQ_crs += __shfl_xor_sync(0xFFFFFFFF, KQ_crs, offset, warp_size); + for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) { + if (offset < warp_size) { + KQ_crs += __shfl_xor_sync(0xFFFFFFFF, KQ_crs, offset, warp_size); + } } } __syncthreads(); - // Write back combined meta data: + if (threadIdx.y % np == 0) { + // Write back combined meta data: #pragma unroll - for (int imeta = 0; imeta < nmeta; ++imeta) { - if (np*cols_per_warp >= warp_size || threadIdx.x < np*cols_per_warp) { - // Combined KQ max scale + rowsum. - meta_ptr[imeta * warp_size * tile_stride/2] = make_float2(KQ_cms[imeta], KQ_crs); + for (int imeta = 0; imeta < nmeta; ++imeta) { + if (np*cols_per_warp >= warp_size || threadIdx.x < np*cols_per_warp) { + // Combined KQ max scale + rowsum. + meta_ptr[imeta * warp_size * tile_stride/2] = make_float2(KQ_cms[imeta], KQ_crs); + } } - } - // Combined KQ max + rowsum. - static_assert(cols_per_warp <= warp_size); - if (needs_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) { - float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols; - dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); - } - if (is_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) { - float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols; - dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); + // Combined KQ max + rowsum. + static_assert(cols_per_warp <= warp_size); + if (needs_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) { + float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols; + dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); + } + if (is_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) { + float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols; + dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); + } } - } else if (np > 1) { - // Warps with threadIdx.y % np == 0 execute a __syncthreads() in the if branch. - // Therefore, all other warps also need to execute a __syncthreads(). - // Otherwise the points at which warps synchronize with each other would become misaligned. - __syncthreads(); } #pragma unroll @@ -1803,7 +1842,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( static constexpr __host__ __device__ bool ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse( const int DKQ, const int DV, const int ncols1, const int ncols2) { return (DKQ == 512 && DV == 512 && ncols1 == 1 && ncols2 == 8) || - (DKQ == 576 && DV == 512 && ncols1 == 1 && ncols2 == 16); + (DKQ == 576 && DV == 512 && ncols1 == 1 && ncols2 == 16) || + (DKQ == 256 && DV == 256 && ncols1 == 1 && ncols2 == 8) || + (DKQ == 256 && DV == 256 && ncols1 == 8 && ncols2 == 8); } template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap, bool V_is_K_view, bool use_sparse> @@ -1837,8 +1878,9 @@ static __global__ void flash_attn_ext_f16( const char * GGML_CUDA_RESTRICT V = V_ptr; const char * GGML_CUDA_RESTRICT mask = mask_ptr; const char * GGML_CUDA_RESTRICT sinks = sinks_ptr; - const int * GGML_CUDA_RESTRICT KV_max = use_sparse ? nullptr : KV_max_ptr; + // sparse: one index list per (sequence, query tile), the live count of each list follows the lists const int * GGML_CUDA_RESTRICT sparse_indices = use_sparse ? KV_max_ptr : nullptr; + const int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr; float * GGML_CUDA_RESTRICT dst = dst_ptr; float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr; @@ -1871,7 +1913,7 @@ static __global__ void flash_attn_ext_f16( #endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING #if defined(AMD_WMMA_AVAILABLE) - if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 128) { + if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256) { NO_DEVICE_CODE; return; } @@ -1903,6 +1945,10 @@ static __global__ void flash_attn_ext_f16( const int iter_j = (ne01.z + (ncols1 - 1)) / ncols1; const int iter_z_gqa = (gqa_ratio + (ncols2 - 1)) / ncols2; + if (use_sparse) { + KV_max = KV_max_ptr + int64_t(iter_j)*ne33*ne11; + } + // kbc == k block continuous, current index in continuous ijk space. int kbc = int64_t(blockIdx.x + 0)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x; const int kbc_stop = int64_t(blockIdx.x + 1)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x; @@ -1932,11 +1978,13 @@ static __global__ void flash_attn_ext_f16( const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV); const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr; - const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*ne31 + jt*ncols1)*ne11 : nullptr; + const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : nullptr; const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f; - if (KV_max) { + if (use_sparse) { + kb0_stop = min(kb0_stop, (KV_max[(sequence % ne33)*iter_j + jt] + nbatch_fa - 1) / nbatch_fa); + } else if (KV_max) { kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa); } constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. @@ -1979,11 +2027,13 @@ static __global__ void flash_attn_ext_f16( const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV); const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr; - const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*ne31 + jt*ncols1)*ne11 : nullptr; + const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : nullptr; const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f; - if (KV_max) { + if (use_sparse) { + kb0_stop = min(kb0_stop, (KV_max[(sequence % ne33)*iter_j + jt] + nbatch_fa - 1) / nbatch_fa); + } else if (KV_max) { kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa); } @@ -2006,7 +2056,7 @@ static __global__ void flash_attn_ext_f16( #endif // defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)) } -bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_tensor * dst, const int ncols1, const int ncols2); template <int DKQ, int DV, int ncols1, int ncols2> void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { @@ -2031,8 +2081,9 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml constexpr bool V_is_K_view = DKQ == 576; // Guaranteed by the kernel selection logic in fattn.cu // KV tile strides must match flash_attn_ext_f16_iter / _process_tile. - const int stride_tile_K = ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_K2, cc); - const int stride_tile_V = V_is_K_view ? stride_tile_K : ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_V2, cc); + const bool swizzled = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols1, ncols2, cc); + const int stride_tile_K = ggml_cuda_fattn_mma_get_stride_tile(nbatch_K2, swizzled); + const int stride_tile_V = V_is_K_view ? stride_tile_K : ggml_cuda_fattn_mma_get_stride_tile(nbatch_V2, swizzled); const size_t nbytes_shared_KV_1stage = nbatch_fa * std::max(stride_tile_K, stride_tile_V) * sizeof(half2); const size_t nbytes_shared_KV_2stage = nbatch_fa * (stride_tile_K + stride_tile_V) * sizeof(half2); const size_t nbytes_shared_Q = ncols * (DKQ/2 + 4) * sizeof(half2); @@ -2059,7 +2110,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml constexpr bool use_logit_softcap = false; #if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, ncols1, ncols2)) { - if (ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ctx, dst)) { + if (ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, ncols1, ncols2)) { constexpr bool use_sparse_kernel = true; fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>; use_sparse = true; diff --git a/ggml/src/ggml-cuda/fattn-swizzle.cuh b/ggml/src/ggml-cuda/fattn-swizzle.cuh deleted file mode 100644 index 44338c8db08d..000000000000 --- a/ggml/src/ggml-cuda/fattn-swizzle.cuh +++ /dev/null @@ -1,126 +0,0 @@ -#pragma once - -#include "common.cuh" -#include "mma.cuh" - -// XOR swizzle for K/V SMEM tiles to avoid bank conflicts without row padding (Turing+ only). -// Stride must be a multiple of 32 half2 columns, otherwise we keep +4 row padding. - -namespace ggml_cuda_fattn_smem_swizzle { - -static __host__ __device__ constexpr bool bank_aligned(const int nbatch_2) { - return nbatch_2 >= 32 && nbatch_2 % 32 == 0; -} - -static __device__ constexpr bool enabled(const int nbatch_2) { -#if defined(TURING_MMA_AVAILABLE) - return bank_aligned(nbatch_2); -#else - GGML_UNUSED(nbatch_2); - return false; -#endif // defined(TURING_MMA_AVAILABLE) -} - -static __host__ bool enabled(const int nbatch_2, const int cc) { -#ifdef GGML_USE_HIP - GGML_UNUSED(nbatch_2); - GGML_UNUSED(cc); - return false; -#else - return turing_mma_available(cc) && bank_aligned(nbatch_2); -#endif // GGML_USE_HIP -} - -static __device__ constexpr int tile_stride(const int nbatch_2) { - return enabled(nbatch_2) ? nbatch_2 : nbatch_2 + 4; -} - -static __host__ int tile_stride(const int nbatch_2, const int cc) { - return enabled(nbatch_2, cc) ? nbatch_2 : nbatch_2 + 4; -} - -// Swizzled byte offset for tile element (row, col_h2), same map used for writes and reads. -template<int stride_h2> -static __device__ __forceinline__ int bytes_rc(const int row, const int col_h2) { - static_assert(bank_aligned(stride_h2), "swizzled tile needs a stride that is a multiple of 32"); - return ((row * stride_h2 + col_h2) * (int) sizeof(half2)) ^ ((row & 7) << 4); -} - -// ldmatrix.x4 via 64-bit generic pointer. -static __device__ __forceinline__ void ldmatrix_x4(int * xi, const half2 * addr) { -#if defined(TURING_MMA_AVAILABLE) - asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];" - : "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3]) - : "l"(addr)); -#else - GGML_UNUSED_VARS(xi, addr); - NO_DEVICE_CODE; -#endif // defined(TURING_MMA_AVAILABLE) -} - -static __device__ __forceinline__ void ldmatrix_x4_trans(int * xi, const half2 * addr) { -#if defined(TURING_MMA_AVAILABLE) - asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];" - : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3]) - : "l"(addr)); -#else - GGML_UNUSED_VARS(xi, addr); - NO_DEVICE_CODE; -#endif // defined(TURING_MMA_AVAILABLE) -} - -// Per-lane swizzled address for one tile<16, 8, half2> ldmatrix: 16 rows, 4 half2 columns per lane. -template<int stride_h2> -static __device__ __forceinline__ const half2 * lane_addr( - const half2 * tile_base, const int base_row, const int base_col_h2, const int I, const int J) { - static_assert(bank_aligned(stride_h2), "swizzled tile needs a stride that is a multiple of 32"); - const int lane_row = threadIdx.x % I; - const int lane_col = (threadIdx.x / I) * (J / 2); - uint32_t byte_off = (uint32_t) ((base_row + lane_row)*stride_h2 + base_col_h2 + lane_col) * (uint32_t) sizeof(half2); - byte_off ^= (uint32_t) (((base_row + lane_row) & 7) << 4); - return (const half2 *) ((const char *) tile_base + byte_off); -} - -template<int stride_h2, bool swz, typename TileT> -static __device__ __forceinline__ void load_ldmatrix( - TileT & t, const half2 * tile_base, const int base_row, const int base_col_h2) { - if constexpr (swz) { - static_assert(std::is_same_v<TileT, ggml_cuda_mma::tile<16, 8, half2>>, - "the swizzled layout is only supported for tile<16, 8, half2>"); - ldmatrix_x4((int *) t.x, lane_addr<stride_h2>(tile_base, base_row, base_col_h2, TileT::I, TileT::J)); - } else { - ggml_cuda_mma::load_ldmatrix(t, tile_base + base_row*stride_h2 + base_col_h2, stride_h2); - } -} - -template<int stride_h2, bool swz, typename TileT> -static __device__ __forceinline__ void load_ldmatrix(TileT & t, const half2 * tile_base, const int off_h2) { - if constexpr (swz) { - load_ldmatrix<stride_h2, swz>(t, tile_base, off_h2 / stride_h2, off_h2 % stride_h2); - } else { - ggml_cuda_mma::load_ldmatrix(t, tile_base + off_h2, stride_h2); - } -} - -template<int stride_h2, bool swz, typename TileT> -static __device__ __forceinline__ void load_ldmatrix_trans( - TileT & t, const half2 * tile_base, const int base_row, const int base_col_h2) { - if constexpr (swz) { - static_assert(std::is_same_v<TileT, ggml_cuda_mma::tile<16, 8, half2>>, - "the swizzled layout is only supported for tile<16, 8, half2>"); - ldmatrix_x4_trans((int *) t.x, lane_addr<stride_h2>(tile_base, base_row, base_col_h2, TileT::I, TileT::J)); - } else { - ggml_cuda_mma::load_ldmatrix_trans(t, tile_base + base_row*stride_h2 + base_col_h2, stride_h2); - } -} - -template<int stride_h2, bool swz, typename TileT> -static __device__ __forceinline__ void load_ldmatrix_trans(TileT & t, const half2 * tile_base, const int off_h2) { - if constexpr (swz) { - load_ldmatrix_trans<stride_h2, swz>(t, tile_base, off_h2 / stride_h2, off_h2 % stride_h2); - } else { - ggml_cuda_mma::load_ldmatrix_trans(t, tile_base + off_h2, stride_h2); - } -} - -} // namespace ggml_cuda_fattn_smem_swizzle diff --git a/ggml/src/ggml-cuda/fattn-vec.cuh b/ggml/src/ggml-cuda/fattn-vec.cuh index 6e81367ab4c2..ff2667ebf192 100644 --- a/ggml/src/ggml-cuda/fattn-vec.cuh +++ b/ggml/src/ggml-cuda/fattn-vec.cuh @@ -336,9 +336,7 @@ static __global__ void flash_attn_ext_vec( #endif // V_DOT2_F32_F16_AVAILABLE } -#ifndef GGML_USE_HIP - __syncwarp(); -#endif // GGML_USE_HIP + ggml_cuda_syncwarp(); #pragma unroll for (int k0 = 0; k0 < WARP_SIZE; k0 += V_cols_per_iter) { diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 935b758ec6c5..fadb1e75e4cb 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -6,10 +6,12 @@ #include "fattn.cuh" #if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +// one list per group of ncols1 queries: a column is selected if any query of the group can see it +template <int ncols1, bool oob> __launch_bounds__(256, 1) static __global__ void flash_attn_mask_to_sparse_indices( - const half * mask_ptr, int32_t * indices_ptr, const int ne30, const int n_kv_max, - const int64_t s31, const int64_t s33) { + const half * mask_ptr, int32_t * indices_ptr, int32_t * counts_ptr, const int ne30, const int n_queries, + const int n_kv_max, const int64_t s31, const int64_t s33) { ggml_cuda_pdl_sync(); constexpr int values_per_lane = 8; @@ -17,10 +19,13 @@ static __global__ void flash_attn_mask_to_sparse_indices( const int warp = tid / WARP_SIZE; const int lane = tid % WARP_SIZE; const int sequence = blockIdx.y; - const int query = blockIdx.x; + const int group = blockIdx.x; - const half * mask = mask_ptr + sequence*s33 + query*s31; - int32_t * indices = indices_ptr + (int64_t(sequence)*gridDim.x + query)*n_kv_max; + const int q0 = group*ncols1; + const int q1 = min(q0 + ncols1, n_queries); + + const half * mask = mask_ptr + sequence*s33 + q0*s31; + int32_t * indices = indices_ptr + (int64_t(sequence)*gridDim.x + group)*n_kv_max; __shared__ int warp_offsets[256/WARP_SIZE]; __shared__ int row_count; @@ -37,7 +42,13 @@ static __global__ void flash_attn_mask_to_sparse_indices( #pragma unroll for (int item = 0; item < values_per_lane; ++item) { const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane; - const bool selected = i < ne30 && isfinite(__half2float(mask[i])); + bool selected = false; + if (i < ne30) { +#pragma unroll + for (int q = 0; q < ncols1; ++q) { + selected |= (!oob || q < q1 - q0) && isfinite(__half2float(mask[q*s31 + i])); + } + } selected_warp[item] = __ballot_sync(0xFFFFFFFF, selected); warp_count += __popc(selected_warp[item]); } @@ -78,10 +89,13 @@ static __global__ void flash_attn_mask_to_sparse_indices( __syncthreads(); } - const int count = row_count; + const int count = min(row_count, n_kv_max); for (int i = count + tid; i < n_kv_max; i += blockDim.x) { indices[i] = -1; } + if (tid == 0) { + counts_ptr[int64_t(sequence)*gridDim.x + group] = count; + } __syncthreads(); // the dependent grid reads indices, signal once the row is complete @@ -90,31 +104,35 @@ static __global__ void flash_attn_mask_to_sparse_indices( #endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) void ggml_cuda_flash_attn_ext_compact_mask( - const ggml_tensor * mask, int32_t * indices, int32_t n_kv_max, cudaStream_t stream) { + const ggml_tensor * mask, int32_t * indices, int32_t * counts, int32_t n_queries, int32_t ncols1, int32_t n_kv_max, cudaStream_t stream) { #if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) - GGML_UNUSED_VARS(mask, indices, n_kv_max, stream); + GGML_UNUSED_VARS(mask, indices, counts, n_queries, ncols1, n_kv_max, stream); GGML_ABORT("sparse flash attention is only supported on NVIDIA CUDA"); #else const int64_t s31 = mask->nb[1] / sizeof(half); const int64_t s33 = mask->nb[3] / sizeof(half); - const dim3 blocks_num(mask->ne[1], mask->ne[3], 1); + const dim3 blocks_num((n_queries + ncols1 - 1)/ncols1, mask->ne[3], 1); const dim3 block_dim(256, 1, 1); const ggml_cuda_kernel_launch_params launch_params(blocks_num, block_dim, 0, stream); - ggml_cuda_kernel_launch(flash_attn_mask_to_sparse_indices, launch_params, - (const half *) mask->data, indices, int(mask->ne[0]), n_kv_max, s31, s33); + // the last group of queries is partial only if ncols1 does not divide n_queries + GGML_ASSERT(ncols1 == 1 || ncols1 == 8); + const auto kernel = ncols1 == 1 ? flash_attn_mask_to_sparse_indices<1, false> : + n_queries % 8 != 0 ? flash_attn_mask_to_sparse_indices<8, true> : + flash_attn_mask_to_sparse_indices<8, false>; + ggml_cuda_kernel_launch(kernel, launch_params, + (const half *) mask->data, indices, counts, int(mask->ne[0]), n_queries, n_kv_max, s31, s33); CUDA_CHECK(cudaGetLastError()); #endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) } -bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { +bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_tensor * dst, const int ncols1, const int ncols2) { #if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) - GGML_UNUSED_VARS(ctx, dst); + GGML_UNUSED_VARS(cc, dst, ncols1, ncols2); return false; #else const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; const ggml_tensor * mask = dst->src[3]; - const int cc = ggml_cuda_info().devices[ctx.device].cc; float max_bias = 0.0f; float logit_softcap = 0.0f; @@ -122,10 +140,14 @@ bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ggml_backend_cuda_context memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); const int32_t n_kv_max = ggml_get_op_params_i32(dst, 4); + + // the dense kernel handles up to 64/ncols2 queries per K/V pass, the single-query gather has to beat that + const int64_t n_gather = (ncols1 == 1 ? std::min<int64_t>(Q->ne[1], 64/ncols2) : ncols1) * (int64_t) n_kv_max; + return GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && mask != nullptr && n_kv_max > 0 && max_bias == 0.0f && logit_softcap == 0.0f && mask->ne[0] == K->ne[1] && mask->ne[1] >= Q->ne[1] && mask->ne[2] == 1 && - K->ne[1] >= std::max<int64_t>(4096, 2LL*n_kv_max); + K->ne[1] >= std::max<int64_t>(4096, 2*n_gather); #endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) } @@ -136,7 +158,9 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con #if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 1, ncols2)) { - if (ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ctx, dst)) { + // a sparse variant at the full tile width gathers the union of its queries once, prefer it for large batches + constexpr bool has_wide_sparse = ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 64/ncols2, ncols2); + if (!(has_wide_sparse && Q->ne[1] > 32/ncols2) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, ncols2)) { ggml_cuda_flash_attn_ext_mma_f16_case<DKQ, DV, 1, ncols2>(ctx, dst); return; } @@ -221,6 +245,24 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con } } + // On RDNA it is preferable to minimize wasted compute vs. duplicate I/O for the mask. + if (amd_wmma_available(cc)) { + if (use_gqa_opt && gqa_ratio % 8 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<DKQ, DV, 8>(ctx, dst); + return; + } + + if (use_gqa_opt && gqa_ratio % 4 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<DKQ, DV, 4>(ctx, dst); + return; + } + + if (use_gqa_opt && gqa_ratio % 2 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<DKQ, DV, 2>(ctx, dst); + return; + } + } + if (use_gqa_opt && gqa_ratio > 4) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<DKQ, DV, 8>(ctx, dst); return; @@ -385,123 +427,118 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg } } -#define FATTN_VEC_CASE(D, type_K, type_V) \ - { \ - const bool type_K_okay = K->type == (type_K) || (K->type == GGML_TYPE_F32 && (type_K) == GGML_TYPE_F16); \ - const bool type_V_okay = V->type == (type_V) || (V->type == GGML_TYPE_F32 && (type_V) == GGML_TYPE_F16); \ - if (Q->ne[0] == (D) && type_K_okay && type_V_okay) { \ - ggml_cuda_flash_attn_ext_vec_case<D, type_K, type_V>(ctx, dst); \ - return; \ - } \ - } \ - -#define FATTN_VEC_CASES_ALL_D(type_K, type_V) \ - FATTN_VEC_CASE( 64, type_K, type_V) \ - FATTN_VEC_CASE(128, type_K, type_V) \ - FATTN_VEC_CASE(256, type_K, type_V) \ +#define FATTN_VEC_CASE(D, type_K_case, type_V_case) \ + if constexpr (GGML_CUDA_FA_##type_K_case##_##type_V_case) { \ + const bool type_K_okay = type_K == GGML_TYPE_##type_K_case || (type_K == GGML_TYPE_F32 && GGML_TYPE_##type_K_case == GGML_TYPE_F16); \ + const bool type_V_okay = type_V == GGML_TYPE_##type_V_case || (type_V == GGML_TYPE_F32 && GGML_TYPE_##type_V_case == GGML_TYPE_F16); \ + if (head_size == (D) && type_K_okay && type_V_okay) { \ + return ggml_cuda_flash_attn_ext_vec_case<D, GGML_TYPE_##type_K_case, GGML_TYPE_##type_V_case>; \ + } \ + } \ + +#define FATTN_VEC_CASES_ALL_D(type_K_case, type_V_case) \ + FATTN_VEC_CASE( 64, type_K_case, type_V_case) \ + FATTN_VEC_CASE(128, type_K_case, type_V_case) \ + FATTN_VEC_CASE(256, type_K_case, type_V_case) \ + +typedef void (* fattn_vec_case_t)(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +// Vector kernel for the given head size and K/V types, nullptr if its template instance was not compiled: +static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, const ggml_type type_K, const ggml_type type_V) { + FATTN_VEC_CASES_ALL_D(F16, F16) + FATTN_VEC_CASES_ALL_D(Q4_0, F16) + FATTN_VEC_CASES_ALL_D(Q4_1, F16) + FATTN_VEC_CASES_ALL_D(Q5_0, F16) + FATTN_VEC_CASES_ALL_D(Q5_1, F16) + FATTN_VEC_CASES_ALL_D(Q8_0, F16) + FATTN_VEC_CASES_ALL_D(BF16, F16) + + FATTN_VEC_CASES_ALL_D(F16, Q4_0) + FATTN_VEC_CASES_ALL_D(Q4_0, Q4_0) + FATTN_VEC_CASES_ALL_D(Q4_1, Q4_0) + FATTN_VEC_CASES_ALL_D(Q5_0, Q4_0) + FATTN_VEC_CASES_ALL_D(Q5_1, Q4_0) + FATTN_VEC_CASES_ALL_D(Q8_0, Q4_0) + FATTN_VEC_CASES_ALL_D(BF16, Q4_0) + + FATTN_VEC_CASES_ALL_D(F16, Q4_1) + FATTN_VEC_CASES_ALL_D(Q4_0, Q4_1) + FATTN_VEC_CASES_ALL_D(Q4_1, Q4_1) + FATTN_VEC_CASES_ALL_D(Q5_0, Q4_1) + FATTN_VEC_CASES_ALL_D(Q5_1, Q4_1) + FATTN_VEC_CASES_ALL_D(Q8_0, Q4_1) + FATTN_VEC_CASES_ALL_D(BF16, Q4_1) + + FATTN_VEC_CASES_ALL_D(F16, Q5_0) + FATTN_VEC_CASES_ALL_D(Q4_0, Q5_0) + FATTN_VEC_CASES_ALL_D(Q4_1, Q5_0) + FATTN_VEC_CASES_ALL_D(Q5_0, Q5_0) + FATTN_VEC_CASES_ALL_D(Q5_1, Q5_0) + FATTN_VEC_CASES_ALL_D(Q8_0, Q5_0) + FATTN_VEC_CASES_ALL_D(BF16, Q5_0) + + FATTN_VEC_CASES_ALL_D(F16, Q5_1) + FATTN_VEC_CASES_ALL_D(Q4_0, Q5_1) + FATTN_VEC_CASES_ALL_D(Q4_1, Q5_1) + FATTN_VEC_CASES_ALL_D(Q5_0, Q5_1) + FATTN_VEC_CASES_ALL_D(Q5_1, Q5_1) + FATTN_VEC_CASES_ALL_D(Q8_0, Q5_1) + FATTN_VEC_CASES_ALL_D(BF16, Q5_1) + + FATTN_VEC_CASES_ALL_D(F16, Q8_0) + FATTN_VEC_CASES_ALL_D(Q4_0, Q8_0) + FATTN_VEC_CASES_ALL_D(Q4_1, Q8_0) + FATTN_VEC_CASES_ALL_D(Q5_0, Q8_0) + FATTN_VEC_CASES_ALL_D(Q5_1, Q8_0) + FATTN_VEC_CASES_ALL_D(Q8_0, Q8_0) + FATTN_VEC_CASES_ALL_D(BF16, Q8_0) + + FATTN_VEC_CASES_ALL_D(F16, BF16) + FATTN_VEC_CASES_ALL_D(Q4_0, BF16) + FATTN_VEC_CASES_ALL_D(Q4_1, BF16) + FATTN_VEC_CASES_ALL_D(Q5_0, BF16) + FATTN_VEC_CASES_ALL_D(Q5_1, BF16) + FATTN_VEC_CASES_ALL_D(Q8_0, BF16) + FATTN_VEC_CASES_ALL_D(BF16, BF16) + + // fork: TurboQuant KV cache types (always compiled, see ggml_cuda_fattn_vec_instances) + FATTN_VEC_CASES_ALL_D(TURBO3_0, TURBO3_0) + FATTN_VEC_CASES_ALL_D(TURBO3_0, Q8_0) + FATTN_VEC_CASES_ALL_D(Q8_0, TURBO3_0) + FATTN_VEC_CASES_ALL_D(TURBO2_0, TURBO2_0) + FATTN_VEC_CASES_ALL_D(TURBO2_0, Q8_0) + FATTN_VEC_CASES_ALL_D(Q8_0, TURBO2_0) + FATTN_VEC_CASES_ALL_D(TURBO3_0, TURBO2_0) + FATTN_VEC_CASES_ALL_D(TURBO2_0, TURBO3_0) + FATTN_VEC_CASES_ALL_D(TURBO4_0, TURBO4_0) + FATTN_VEC_CASES_ALL_D(TURBO4_0, Q8_0) + FATTN_VEC_CASES_ALL_D(Q8_0, TURBO4_0) + FATTN_VEC_CASES_ALL_D(TURBO4_0, TURBO3_0) + FATTN_VEC_CASES_ALL_D(TURBO3_0, TURBO4_0) + FATTN_VEC_CASES_ALL_D(TURBO4_0, TURBO2_0) + FATTN_VEC_CASES_ALL_D(TURBO2_0, TURBO4_0) + + return nullptr; +} static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - ggml_tensor * Q = dst->src[0]; - ggml_tensor * K = dst->src[1]; - ggml_tensor * V = dst->src[2]; - -#ifdef GGML_CUDA_FA_ALL_QUANTS - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_F16) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q4_0) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q4_1) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q5_0) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q5_1) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q8_0) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_BF16) -#else - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_BF16) -#endif // GGML_CUDA_FA_ALL_QUANTS - - // TurboQuant3 KV cache types (always enabled) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0) - - // Mixed turbo3/q8_0 KV cache types - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0) - - // TurboQuant2 KV cache types (always enabled) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0) - - // Mixed turbo2/q8_0 KV cache types - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO2_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0) - - // Mixed turbo3/turbo2 KV cache types - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO2_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO3_0) - - // TurboQuant4 KV cache types (always enabled) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0) - - // Mixed turbo4/q8_0 KV cache types - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0) - - // Mixed turbo4/turbo3 KV cache types - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO3_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO4_0) - - // Mixed turbo4/turbo2 KV cache types - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO2_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO4_0) + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; - GGML_ABORT("fatal error"); + fattn_vec_case_t vec_case = ggml_cuda_get_fattn_vec_case(Q->ne[0], K->type, V->type); + if (vec_case == nullptr) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("%s: no FlashAttention vector kernel compiled for K/V types %s-%s, converting K and V to f16 instead (slow). " + "Add \"%s-%s\" to GGML_CUDA_FA_QUANTS to compile it.\n", + __func__, ggml_type_name(K->type), ggml_type_name(V->type), ggml_type_name(K->type), ggml_type_name(V->type)); + warned = true; + } + vec_case = ggml_cuda_get_fattn_vec_case(Q->ne[0], GGML_TYPE_F16, GGML_TYPE_F16); + } + GGML_ASSERT(vec_case != nullptr); + vec_case(ctx, dst); } // Best FlashAttention kernel for a specific GPU: @@ -512,20 +549,17 @@ enum best_fattn_kernel { BEST_FATTN_KERNEL_MMA_F16 = 400, }; -static bool ggml_cuda_fattn_kv_type_supported(ggml_type type) { +// K/V types for which there is a vector kernel template instance, other kernels convert these to f16: +static bool ggml_cuda_fattn_kv_type_supported(const ggml_type type) { switch (type) { case GGML_TYPE_F32: case GGML_TYPE_F16: - return true; + case GGML_TYPE_BF16: + case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: -#ifndef GGML_CUDA_FA_ALL_QUANTS - return false; -#endif // GGML_CUDA_FA_ALL_QUANTS - case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: - case GGML_TYPE_BF16: return true; // fork: TurboQuant KV cache types. Their VEC kernels are only // instantiated for D in {64, 128, 256}, which the caller checks. @@ -623,18 +657,6 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_NONE; } -#ifndef GGML_CUDA_FA_ALL_QUANTS - if (K->type != V->type) { - // Allow mixed turbo KV types (any combination of turbo2, turbo3, q8_0) - auto is_turbo = [](ggml_type t) { - return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || t == GGML_TYPE_TURBO4_0 || t == GGML_TYPE_Q8_0; - }; - if (!is_turbo(K->type) || !is_turbo(V->type)) { - return BEST_FATTN_KERNEL_NONE; - } - } -#endif // GGML_CUDA_FA_ALL_QUANTS - if (!ggml_cuda_fattn_kv_type_supported(K->type) || !ggml_cuda_fattn_kv_type_supported(V->type)) { return BEST_FATTN_KERNEL_NONE; } @@ -675,7 +697,12 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { if (can_use_vector_kernel) { if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { - if (cc >= GGML_CUDA_CC_ADA_LOVELACE && Q->ne[1] == 1 && Q->ne[3] == 1 && !(gqa_ratio > 4 && K->ne[1] >= 8192)) { + // the sparse gather exists only in the MMA kernel: (DKQ, DV, 1, 8) with GQA > 4 + const bool sparse_decode = gqa_opt_applies && gqa_ratio > 4 && + ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(K->ne[0], V->ne[0], 1, 8) && + ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 8); + if (!sparse_decode && cc >= GGML_CUDA_CC_ADA_LOVELACE && Q->ne[1] == 1 && Q->ne[3] == 1 && + !(gqa_ratio > 4 && (Q->ne[0] >= 256 || K->ne[1] >= 8192))) { return BEST_FATTN_KERNEL_VEC; } } else { @@ -761,8 +788,9 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const } } - // AMD WMMA is always faster than the tile kernel if the full tile width of 16 can be utilized. - if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 128) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[1] * gqa_ratio_eff > 8) { + // AMD WMMA is faster than the tile kernel if the wide tiles with high arithmetic intensity can be utilized. + if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72 && + Q->ne[1] * gqa_ratio_eff > (Q->ne[0] <= 128 ? 8 : 16)) { return BEST_FATTN_KERNEL_MMA_F16; } @@ -786,6 +814,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * dst) { GGML_ASSERT(dst->op == GGML_OP_FLASH_ATTN_EXT); + const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; const ggml_tensor * V = dst->src[2]; @@ -803,10 +832,11 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d need_f16_K = true; need_f16_V = true; break; - case BEST_FATTN_KERNEL_VEC: - need_f16_K = K->type == GGML_TYPE_F32; - need_f16_V = V->type == GGML_TYPE_F32; - break; + case BEST_FATTN_KERNEL_VEC: { + const bool f16_fallback = ggml_cuda_get_fattn_vec_case(Q->ne[0], K->type, V->type) == nullptr; + need_f16_K = K->type == GGML_TYPE_F32 || f16_fallback; + need_f16_V = V->type == GGML_TYPE_F32 || f16_fallback; + } break; case BEST_FATTN_KERNEL_NONE: break; } diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 49d97f41ea5a..1ac298a283a0 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -430,6 +430,7 @@ static int ggml_cuda_parse_id(char devName[]) { } archNum += archMajor * 0x100; archNum += archMinor; + return archNum; } #endif // defined(GGML_USE_HIP) @@ -575,11 +576,7 @@ static ggml_cuda_device_info ggml_cuda_init() { info.default_tensor_split[id] = total_vram; total_vram += device_vram; -#if defined(GGML_USE_HIP) - info.devices[id].integrated = prop.integrated; -#else info.devices[id].integrated = false; // Temporarily disabled due to issues with corrupted output (e.g. #15034) -#endif info.devices[id].nsm = prop.multiProcessorCount; info.devices[id].smpb = prop.sharedMemPerBlock; info.devices[id].warp_size = prop.warpSize; @@ -3126,16 +3123,19 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml } } + const int cc = ggml_cuda_info().devices[ctx.device].cc; ggml_type compute_type = src0->type; if (ggml_is_quantized(compute_type)) { - compute_type = fast_fp16_hardware_available(ggml_cuda_info().devices[ctx.device].cc) ? GGML_TYPE_F16 : GGML_TYPE_F32; - } else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(ggml_cuda_info().devices[ctx.device].cc)) { - compute_type = GGML_TYPE_F32; - } else if (compute_type == GGML_TYPE_BF16 && !bf16_mma_hardware_available(ggml_cuda_info().devices[ctx.device].cc)) { - // Mirrors the F16 arm above. Without BF16 tensor cores cuBLAS emulates the BF16 compute - // type extremely slowly; F32 (not F16) is the fallback because the BF16 -> compute-type - // conversion must stay exact - F16 would round the exponent range away. + compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32; + } else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) { compute_type = GGML_TYPE_F32; + } else if (compute_type == GGML_TYPE_BF16 && !fast_bf16_hardware_available(cc)) { + if (GGML_CUDA_CC_IS_AMD(cc) && src1->ne[1] > 32) { + compute_type = GGML_TYPE_F32; + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && src1->ne[1] > (cc >= GGML_CUDA_CC_VOLTA ? 8 : 128)) { + compute_type = GGML_TYPE_F32; + } } if (dst->op_params[0] == GGML_PREC_F32) { compute_type = GGML_TYPE_F32; @@ -8774,6 +8774,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_cuda_topk_moe_args args; const bool can_fuse = ggml_cuda_topk_moe_fusion(cgraph, i, args); std::vector<ggml_op> ops; + ops.reserve(13); // max ops; avoids gcc -Wstringop-overflow false positive if (can_fuse) { const ggml_tensor * logits = node->src[0]; @@ -10605,24 +10606,91 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph // upstream: record allocation dependencies for the fused MoE weighted reduction static const bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION")); - if (!disable_fusion) { - for (int i = 0; i < cgraph->n_nodes; ++i) { - if (cgraph->nodes[i]->op != GGML_OP_MUL) { - continue; + + auto add_alloc_deps = [&](size_t start, size_t last_node) { + + for (size_t i = start; i < last_node; ++i) { + params->add_alloc_dep(params->user_data, cgraph->nodes[i], cgraph->nodes[last_node]); + + for (int j = 0; j < GGML_MAX_SRC; ++j) { + if (cgraph->nodes[i]->src[j]) { + params->add_alloc_dep(params->user_data, cgraph->nodes[i]->src[j], cgraph->nodes[last_node]); + } } + } + }; + if (!disable_fusion) { + // add alloc deps for performance positive fusions. This may increase the overall compute buffer size. + // TODO: consolidate fusion paths in graph_optimize and graph_compute + for (int i = 0; i < cgraph->n_nodes; ++i) { ggml_cuda_moe_weighted_reduction_match match; - if (!ggml_cuda_match_moe_weighted_reduction(cgraph, i, match)) { - continue; + if (ggml_cuda_match_moe_weighted_reduction(cgraph, i, match)) { + params->add_alloc_dep(params->user_data, const_cast<ggml_tensor *>(match.experts), match.dst); + params->add_alloc_dep(params->user_data, const_cast<ggml_tensor *>(match.weights), match.dst); + if (match.expert_scale != nullptr) { + params->add_alloc_dep( + params->user_data, const_cast<ggml_tensor *>(match.expert_scale), match.dst); + } + i += match.node_count - 1; } - params->add_alloc_dep(params->user_data, const_cast<ggml_tensor *>(match.experts), match.dst); - params->add_alloc_dep(params->user_data, const_cast<ggml_tensor *>(match.weights), match.dst); - if (match.expert_scale != nullptr) { - params->add_alloc_dep( - params->user_data, const_cast<ggml_tensor *>(match.expert_scale), match.dst); + if (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX || + cgraph->nodes[i]->op == GGML_OP_ARGSORT) { + ggml_cuda_topk_moe_args args; + const bool can_fuse = ggml_cuda_topk_moe_fusion(cgraph, i, args); + std::vector<ggml_op> ops; + ops.reserve(13); // max ops; avoids gcc -Wstringop-overflow false positive + + const ggml_tensor * node = cgraph->nodes[i]; + + if (can_fuse) { + const ggml_tensor * logits = node->src[0]; + ggml_tensor * weights = nullptr; + ggml_tensor * ids = nullptr; + + if (!args.delayed_softmax) { + int out_nodes[2]; // nodes which can't be elided + + if (args.sigmoid) { + ops.insert(ops.end(), { GGML_OP_UNARY }); + } else if (args.sqrt_softplus) { + ops.insert(ops.end(), { GGML_OP_UNARY, GGML_OP_SQRT }); + } else { + ops.insert(ops.end(), { GGML_OP_SOFT_MAX }); + } + const int i_probs = i + (int) ops.size() - 1; // last node of the gating activation + + if (args.prob_bias) { + ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, + GGML_OP_GET_ROWS }); + out_nodes[0] = i_probs + 4; + } else { + ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS }); + out_nodes[0] = i_probs + 3; + } + ids = cgraph->nodes[out_nodes[0]]; + + if (args.norm) { + ops.insert(ops.end(), + { GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE }); + } + if (args.scale) { + ops.insert(ops.end(), { GGML_OP_SCALE }); + } + + weights = cgraph->nodes[i + ops.size() - 1]; + out_nodes[1] = i + ops.size() - 1; + + if (ggml_can_fuse_subgraph(cgraph, i, ops.size(), ops.data(), out_nodes, 2) && + ggml_cuda_should_use_topk_moe(node, logits, weights, ids)) { + + add_alloc_deps(i, i + ops.size()); + i += ops.size() - 1; + } + } + } } - i += match.node_count - 1; } } @@ -11269,6 +11337,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g if (b->type == GGML_TYPE_F16 && a->type != GGML_TYPE_F16) { return false; } + if (op->op == GGML_OP_MUL_MAT_ID && ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { + return false; + } #ifdef GGML_USE_MUSA const int cc = ggml_cuda_info().devices[dev_ctx->device].cc; if (b->ne[2]*b->ne[3] > 1 && !ggml_is_transposed(a) && !ggml_is_transposed(b)) { @@ -11757,10 +11828,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return false; } break; case GGML_OP_DUP: - { - ggml_type src0_type = op->src[0]->type; - return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16; - } break; + return true; case GGML_OP_ARGMAX: case GGML_OP_COUNT_EQUAL: { @@ -11942,7 +12010,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return true; #endif case GGML_OP_SUM_ROWS: + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]); case GGML_OP_MEAN: + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]); case GGML_OP_GROUP_NORM: return ggml_is_contiguous(op->src[0]); case GGML_OP_PAD: @@ -11971,7 +12041,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g op->type == GGML_TYPE_F32; case GGML_OP_DSV4_HC_POST: return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && - op->src[2]->type == GGML_TYPE_F32 && op->src[3]->type == GGML_TYPE_F32 && + op->src[2]->type == GGML_TYPE_F32 && (op->src[3] == nullptr || op->src[3]->type == GGML_TYPE_F32) && op->type == GGML_TYPE_F32; case GGML_OP_FLASH_ATTN_EXT: return ggml_cuda_flash_attn_ext_supported(dev_ctx->device, op); @@ -12153,8 +12223,8 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t features.push_back({ "USE_GRAPHS", "1" }); #endif - #ifdef GGML_CUDA_FA_ALL_QUANTS - features.push_back({ "FA_ALL_QUANTS", "1" }); + #ifdef GGML_CUDA_FA_QUANTS + features.push_back({ "FA_QUANTS", GGML_CUDA_FA_QUANTS }); #endif { diff --git a/ggml/src/ggml-cuda/im2col.cu b/ggml/src/ggml-cuda/im2col.cu index 28c79ab462e2..d377f2856439 100644 --- a/ggml/src/ggml-cuda/im2col.cu +++ b/ggml/src/ggml-cuda/im2col.cu @@ -7,40 +7,41 @@ template <typename T> static __global__ void im2col_kernel( const float * x, T * dst, int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, - int64_t IC_IH_IW, int64_t IH_IW, int64_t N_OH, int64_t KH_KW, int64_t IC_KH_KW, + int64_t N, int64_t IC_IH_IW, int64_t IH_IW, int64_t N_OH, int64_t KH_KW, int64_t IC_KH_KW, int s0, int s1, int p0, int p1, int d0, int d1) { - const int64_t i = threadIdx.x + blockIdx.x * blockDim.x; - if (i >= IC_KH_KW) { - return; - } - - const int64_t iic = i / (KH_KW); - const int64_t rem = i - iic * KH_KW; - const int64_t ikh = rem / KW; - const int64_t ikw = rem - ikh * KW; - - for (int64_t iow = blockIdx.y; iow < OW; iow += MAX_GRIDDIM_Y) { - for (int64_t iz = blockIdx.z; iz < N_OH; iz += MAX_GRIDDIM_Z) { - const int64_t in = iz / OH; - const int64_t ioh = iz - in * OH; - - const int64_t iiw = iow * s0 + ikw * d0 - p0; - const int64_t iih = ioh * s1 + ikh * d1 - p1; - - const int64_t offset_dst = - ((in * OH + ioh) * OW + iow) * IC_KH_KW + iic * KH_KW + ikh * KW + ikw; - - if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) { - dst[offset_dst] = 0.0f; - } else { - const int64_t offset_src = iic * IC_IH_IW + in * IH_IW; - dst[offset_dst] = x[offset_src + iih * IW + iiw]; + const int tid = threadIdx.x; + + const int64_t total_channels = IC * KH * KW; + const int threads_per_pos = blockDim.x; + const int64_t start_ch = tid; + const int64_t stride_ch = threads_per_pos; + + for (int64_t iow = blockIdx.x; iow < OW; iow += MAX_GRIDDIM_Y) { + for (int64_t iz = blockIdx.y; iz < N_OH; iz += MAX_GRIDDIM_Z) { + const int64_t in = iz / OH; + const int64_t ioh = iz - in * OH; + + for (int64_t iic_khw = start_ch; iic_khw < total_channels; iic_khw += stride_ch) { + const int64_t iic = iic_khw / KH_KW; + const int64_t rem = iic_khw - iic * KH_KW; + const int64_t ikh = rem / KW; + const int64_t ikw = rem - ikh * KW; + + const int64_t iiw = iow * s0 + ikw * d0 - p0; + const int64_t iih = ioh * s1 + ikh * d1 - p1; + + const int64_t offset_dst = + ((in * OH + ioh) * OW + iow) * IC_KH_KW + iic * KH_KW + ikh * KW + ikw; + + if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) { + dst[offset_dst] = 0.0f; + } else { + const int64_t offset_src = iic * IC_IH_IW + in * IH_IW; + dst[offset_dst] = x[offset_src + iih * IW + iiw]; + } } } } - - GGML_UNUSED(IC); - GGML_UNUSED(KH); } // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW] @@ -50,13 +51,15 @@ static void im2col_cuda(const float * x, T* dst, int64_t N, int64_t IC_IH_IW, int64_t IH_IW, int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) { const int64_t IC_KH_KW = IC * KH * KW; - const int64_t num_blocks = (IC_KH_KW + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE; const int64_t N_OH = N * OH; const int64_t KH_KW = KW*KH; - dim3 block_nums(num_blocks, MIN(OW, MAX_GRIDDIM_Y), MIN(N_OH, MAX_GRIDDIM_Z)); - im2col_kernel<<<block_nums, MIN(IC_KH_KW, CUDA_IM2COL_BLOCK_SIZE) , 0, stream>>>(x, dst, IC, IW, IH, OH, OW, KW, KH, - IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW, - s0, s1, p0, p1, d0, d1); + const int threads_per_block = MIN((int)IC_KH_KW, CUDA_IM2COL_BLOCK_SIZE); + dim3 block_nums(MIN(OW, MAX_GRIDDIM_Y), MIN(N_OH, MAX_GRIDDIM_Z)); + + im2col_kernel<<<block_nums, threads_per_block, 0, stream>>>( + x, dst, IC, IW, IH, OH, OW, KW, KH, + N, IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW, + s0, s1, p0, p1, d0, d1); } static void im2col_cuda_f16(const float * x, half * dst, diff --git a/ggml/src/ggml-cuda/mean.cu b/ggml/src/ggml-cuda/mean.cu index 8231ca817862..4b79e677844b 100644 --- a/ggml/src/ggml-cuda/mean.cu +++ b/ggml/src/ggml-cuda/mean.cu @@ -27,7 +27,7 @@ void ggml_cuda_op_mean(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous_rows(src0)); const int64_t ncols = src0->ne[0]; const int64_t nrows = ggml_nrows(src0); @@ -74,13 +74,20 @@ void ggml_cuda_op_mean(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { // Heuristic for block size selection to optimize occupancy. // See discussion in: https://github.com/ggml-org/llama.cpp/pull/15132 + dim3 block_dims; if ((nrows / nsm) < 2) { - const dim3 block_dims(512, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); - ggml_cuda_kernel_launch(reduce_rows_f32</*norm=*/true>, launch_params, src0_d, dst_d, ncols); + block_dims = dim3(512, 1, 1); } else { - const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + block_dims = dim3(ncols < 1024 ? 32 : 128, 1, 1); + } + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + + if (ggml_is_contiguous(src0)) { ggml_cuda_kernel_launch(reduce_rows_f32</*norm=*/true>, launch_params, src0_d, dst_d, ncols); + return; } + + const char * src0_d_bytes = (const char *) src0->data; + ggml_cuda_kernel_launch(reduce_rows_f32_strided</*norm=*/true>, launch_params, src0_d_bytes, dst_d, ncols, + src0->ne[1], src0->ne[2], src0->nb[1], src0->nb[2], src0->nb[3]); } diff --git a/ggml/src/ggml-cuda/mma.cuh b/ggml/src/ggml-cuda/mma.cuh index e47a02cd2216..d18d50a6cd51 100644 --- a/ggml/src/ggml-cuda/mma.cuh +++ b/ggml/src/ggml-cuda/mma.cuh @@ -787,6 +787,20 @@ namespace ggml_cuda_mma { } } + // Byte offset of tile element (i, j). If swz, XOR swizzle it to avoid bank conflicts without row padding. + template <bool swz, typename T> + static __device__ __forceinline__ int swizzle_bytes(const int i, const int j, const int stride) { + static_assert(!swz || sizeof(T) == 4, "swizzled tiles need 32 bit elements"); + const int off = (i*stride + j) * (int) sizeof(T); + return swz ? off ^ ((i & 7) << 4) : off; + } + + template <bool swz, typename T> + static __device__ __forceinline__ const T * swizzle( + const T * __restrict__ tile_base, const int i, const int j, const int stride) { + return (const T *) ((const char *) tile_base + swizzle_bytes<swz, T>(i, j, stride)); + } + template <typename T> static __device__ __forceinline__ void load_ldmatrix( tile<8, 8, T> & t, const T * __restrict__ xs0, const int stride) { @@ -863,6 +877,29 @@ namespace ggml_cuda_mma { #endif // TURING_MMA_AVAILABLE } + // Load from tile element (i0, j0), swz tells if the tile is stored swizzled. + template <bool swz, int I, int J, typename T, data_layout dl> + static __device__ __forceinline__ void load_ldmatrix( + tile<I, J, T, dl> & t, const T * __restrict__ tile_base, const int i0, const int j0, const int stride) { + if constexpr (!swz) { + load_ldmatrix(t, tile_base + i0*stride + j0, stride); + return; + } +#if defined(TURING_MMA_AVAILABLE) + static_assert(I == 16, "bad tile width"); + static_assert(J == 8, "bad tile height"); + const int i = i0 + threadIdx.x % t.I; + const int j = j0 + (threadIdx.x / t.I) * (t.J / 2); + int * xi = (int *) t.x; + asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];" + : "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3]) + : "l"(swizzle<true>(tile_base, i, j, stride))); +#else + GGML_UNUSED_VARS(t, tile_base, i0, j0, stride); + NO_DEVICE_CODE; +#endif // defined(TURING_MMA_AVAILABLE) + } + static __device__ __forceinline__ void load_ldmatrix( tile<8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int stride) { ggml_cuda_memcpy_1<4*sizeof(half2)>(t.x, xs0 + t.get_i(0)*stride); @@ -922,6 +959,29 @@ namespace ggml_cuda_mma { #endif // TURING_MMA_AVAILABLE } + // Load from tile element (i0, j0), swz tells if the tile is stored swizzled. + template <bool swz, int I, typename T, data_layout dl> + static __device__ __forceinline__ void load_ldmatrix_trans( + tile<I, 8, T, dl> & t, const T * __restrict__ tile_base, const int i0, const int j0, const int stride) { + if constexpr (!swz) { + load_ldmatrix_trans(t, tile_base + i0*stride + j0, stride); + return; + } +#if defined(TURING_MMA_AVAILABLE) + static_assert(I == 16, "bad tile width"); + static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout"); + const int i = i0 + threadIdx.x % t.I; + const int j = j0 + (threadIdx.x / t.I) * (t.J / 2); + int * xi = (int *) t.x; + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];" + : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3]) + : "l"(swizzle<true>(tile_base, i, j, stride))); +#else + GGML_UNUSED_VARS(t, tile_base, i0, j0, stride); + NO_DEVICE_CODE; +#endif // defined(TURING_MMA_AVAILABLE) + } + static __device__ __forceinline__ void mma( tile<16, 8, int> & D, const tile<16, 4, int> & A, const tile<8, 4, int> & B) { #ifdef TURING_MMA_AVAILABLE diff --git a/ggml/src/ggml-cuda/mmf.cuh b/ggml/src/ggml-cuda/mmf.cuh index d55cc1ec7b52..879a86527507 100644 --- a/ggml/src/ggml-cuda/mmf.cuh +++ b/ggml/src/ggml-cuda/mmf.cuh @@ -143,6 +143,7 @@ static __global__ void mul_mat_f( if (threadIdx.x == 0) { slot_map[j] = -1; } + ggml_cuda_syncwarp(); if (col_base + j >= ncols_dst_total) { continue; @@ -171,10 +172,12 @@ static __global__ void mul_mat_f( tile_A A[ntA][warp_size / tile_A::J]; #pragma unroll for (int itA = 0; itA < ntA; ++itA) { + ggml_cuda_syncwarp(); #pragma unroll for (int i = 0; i < tile_A::I; ++i) { tile_xy[i*tile_k_padded + threadIdx.x] = x[(itA*tile_A::I + i)*stride_row + col]; } + ggml_cuda_syncwarp(); #pragma unroll for (int k0 = 0; k0 < warp_size; k0 += tile_A::J) { load_ldmatrix(A[itA][k0/tile_A::J], tile_xy + k0, tile_k_padded); @@ -183,6 +186,7 @@ static __global__ void mul_mat_f( #pragma unroll for (int itB = 0; itB < ntB; ++itB) { + ggml_cuda_syncwarp(); if constexpr (std::is_same_v<T, float>) { #pragma unroll for (int j0 = 0; j0 < tile_B::I; ++j0) { @@ -212,6 +216,7 @@ static __global__ void mul_mat_f( } else { static_assert(std::is_same_v<T, void>, "unsupported type"); } + ggml_cuda_syncwarp(); #pragma unroll for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) { tile_B B; @@ -229,6 +234,8 @@ static __global__ void mul_mat_f( if (nwarps > 1) { __syncthreads(); + } else { + ggml_cuda_syncwarp(); } #pragma unroll for (int itB = 0; itB < ntB; ++itB) { @@ -245,6 +252,8 @@ static __global__ void mul_mat_f( if (nwarps > 1) { __syncthreads(); + } else { + ggml_cuda_syncwarp(); } #pragma unroll @@ -382,10 +391,12 @@ static __global__ void mul_mat_f_ids( tile_A A[ntA][warp_size / tile_A::J]; #pragma unroll for (int itA = 0; itA < ntA; ++itA) { + ggml_cuda_syncwarp(); #pragma unroll for (int i = 0; i < tile_A::I; ++i) { tile_xy[i*tile_k_padded + threadIdx.x] = x[(itA*tile_A::I + i)*stride_row + col]; } + ggml_cuda_syncwarp(); #pragma unroll for (int k0 = 0; k0 < warp_size; k0 += tile_A::J) { load_ldmatrix(A[itA][k0/tile_A::J], tile_xy + k0, tile_k_padded); @@ -419,6 +430,7 @@ static __global__ void mul_mat_f_ids( int next_buf = 1; #pragma unroll for (int itB = 0; itB < ntB; ++itB) { + ggml_cuda_syncwarp(); #pragma unroll for (int j0 = 0; j0 < tile_B::I; ++j0) { tile_xy[j0*tile_k_padded + threadIdx.x] = vals_buf[curr_buf][j0]; @@ -428,6 +440,7 @@ static __global__ void mul_mat_f_ids( gather_tile(itB + 1, vals_buf[next_buf]); } + ggml_cuda_syncwarp(); #pragma unroll for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) { tile_B B; @@ -472,6 +485,7 @@ static __global__ void mul_mat_f_ids( int next_buf = 1; #pragma unroll for (int itB = 0; itB < ntB; ++itB) { + ggml_cuda_syncwarp(); #pragma unroll for (int j0 = 0; j0 < tile_B::I; ++j0) { const float2 tmp = vals_buf[curr_buf][j0]; @@ -482,6 +496,7 @@ static __global__ void mul_mat_f_ids( gather_tile(itB + 1, vals_buf[next_buf]); } + ggml_cuda_syncwarp(); #pragma unroll for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) { tile_B B; @@ -507,6 +522,8 @@ static __global__ void mul_mat_f_ids( if (nwarps > 1) { __syncthreads(); + } else { + ggml_cuda_syncwarp(); } #pragma unroll for (int itB = 0; itB < ntB; ++itB) { @@ -523,6 +540,8 @@ static __global__ void mul_mat_f_ids( if (nwarps > 1) { __syncthreads(); + } else { + ggml_cuda_syncwarp(); } #pragma unroll diff --git a/ggml/src/ggml-cuda/mmid.cu b/ggml/src/ggml-cuda/mmid.cu index ed0851dcf8dd..0b222e63ac7d 100644 --- a/ggml/src/ggml-cuda/mmid.cu +++ b/ggml/src/ggml-cuda/mmid.cu @@ -101,6 +101,7 @@ static __global__ void mm_ids_helper( } } nex_prev = warp_reduce_sum<warp_size>(nex_prev); + ggml_cuda_syncwarp(); for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) { const mm_ids_helper_store store_it = store[itc]; diff --git a/ggml/src/ggml-cuda/mmq-config-gcn.cuh b/ggml/src/ggml-cuda/mmq-config-gcn.cuh new file mode 100644 index 000000000000..24af2ef2b05b --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-gcn.cuh @@ -0,0 +1,281 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_gcn(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 3, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 601e83a6844a..8ddc47d74b7b 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -414,7 +414,7 @@ void ggml_cuda_mul_mat_q( ne00, ne01, ne1, s01, ne11, s1, ne02, ne12, s02, s12, s2, ne03, ne13, s03, s13, s3, - ne1}; + ne1, ne1}; args.expert_ptrs = routed_expert_ptrs; ggml_cuda_mul_mat_q_switch_type(ctx, args, stream); return; @@ -490,6 +490,13 @@ void ggml_cuda_mul_mat_q( ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int)); const int64_t s13 = ne12*s12; + // Each expert only sees ne12*n_expert_used/ne02 tokens on average. + // On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12. + int64_t ncols_opt = ne12; + if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) { + ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02; + } + // Note that ne02 is used instead of ne12 because the number of y channels determines the z dimension of the CUDA grid. mmq_args args = { src0_d, src0->type, (const int *) src1_q8_1.get(), ids_dst.get(), expert_bounds.get(), dst_d, @@ -497,7 +504,7 @@ void ggml_cuda_mul_mat_q( ne00, ne01, ne_get_rows, s01, ne_get_rows, s1, ne02, ne02, s02, s12, s2, ne03, ne13, s03, s13, s3, - ne12}; + ne12, ncols_opt}; args.force_mm_id = force_mm_id; args.expert_ptrs = routed_expert_ptrs; @@ -636,10 +643,10 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t return ne11 < MMQ_DP4A_MAX_BATCH_SIZE; } - // gfx900 (Vega 10) lacks native dp4a, loses to dequant + hipBLAS + // gfx900 (Vega 10), gfx909, and gfx90c lack native dp4a, losing to dequant + hipBLAS // for dense matrices; keep MMQ only for MoE, where the // hipBLAS path is much slower. - if (cc == GGML_CUDA_CC_VEGA) { + if (cc == GGML_CUDA_CC_VEGA || GGML_CUDA_CC_IS_GCN_APU(cc)) { return n_experts > 0; } diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index c62c619b2f83..21196f49422e 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -223,6 +223,7 @@ struct ggml_cuda_mmq_config { #include "mmq-config-ampere.cuh" #include "mmq-config-blackwell.cuh" +#include "mmq-config-gcn.cuh" #include "mmq-config-cdna.cuh" #include "mmq-config-rdna2.cuh" #include "mmq-config-rdna3.cuh" @@ -233,6 +234,9 @@ struct ggml_cuda_mmq_config { static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc) { if (GGML_CUDA_CC_IS_AMD(cc)) { + if (GGML_CUDA_CC_IS_GCN(cc)) { + return ggml_cuda_mmq_get_config_gcn(type, J, fallback); + } if (GGML_CUDA_CC_IS_CDNA(cc)) { return ggml_cuda_mmq_get_config_cdna(type, J, fallback); } @@ -261,7 +265,9 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback) { #ifdef GGML_USE_HIP -#ifdef CDNA +#ifdef GCN + return ggml_cuda_mmq_get_config_gcn(type, J, fallback); +#elif defined(CDNA) return ggml_cuda_mmq_get_config_cdna(type, J, fallback); #elif defined(RDNA4) return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); @@ -1419,6 +1425,7 @@ struct mmq_args { int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst; int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst; int64_t ncols_max; + int64_t ncols_opt; // value to optimize the tile size against, launch grid still uses ncols_max bool force_mm_id = false; // Optional per-expert weight pointer array for routing-aware weight // paging on consolidated MoE (MAD-88). When non-null, the kernel reads @@ -1542,7 +1549,9 @@ static void mul_mat_q_switch_J_impl(ggml_backend_cuda_context & ctx, const mmq_a const long value = std::strtol(env, nullptr, 10); return value < 0 ? (int64_t) MMQ_FORCE_MM_REFERENCE_TOKENS : (int64_t) value; }(); - const int64_t ncols_max = args.force_mm_id && pin_ref_tokens > 0 ? pin_ref_tokens : args.ncols_max; + // fork: a pinned expert mul_mat keeps its fixed reference width (width-invariant tiling); + // everything else sizes tiles against upstream's ncols_opt + const int64_t ncols_tile = args.force_mm_id && pin_ref_tokens > 0 ? pin_ref_tokens : args.ncols_opt; for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) { const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc); @@ -1554,7 +1563,7 @@ static void mul_mat_q_switch_J_impl(ggml_backend_cuda_context & ctx, const mmq_a continue; } - const int ntiles_x = (ncols_max + config.J - 1) / config.J; + const int ntiles_x = (ncols_tile + config.J - 1) / config.J; if (ntiles_x < ntiles_J_best) { J_best = J; diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index a8a0137462e7..98a5cf2ef384 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -8,6 +8,35 @@ #include <cstdlib> #include <type_traits> +// only enabled on DGX Spark, where it is a gain on every type below. On the higher-bandwidth parts the kernel +// has little exposed latency left to hide and the extra requests cost more than they save. +// For perf data, see https://github.com/ggml-org/llama.cpp/pull/26705#issuecomment-5569335031 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK +// returns true only for those quants that benefit from prefetch and false otherwise +static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) { + switch (type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q8_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + return true; + default: + return false; + } +} + +static __device__ __forceinline__ void mmvq_prefetch_l2(const void * p) { + asm volatile("prefetch.global.L2 [%0];" :: "l"(p)); +} +#endif + typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs); static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) { @@ -378,9 +407,6 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { return ne11 <= 4; case GGML_TYPE_Q3_K: return ne11 <= 6; - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - return ne11 <= 7; default: return ne11 <= MMVQ_MAX_BATCH_SIZE; } @@ -390,8 +416,9 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: return ne11 <= 5; + case GGML_TYPE_Q5_K: + return ne11 <= 6; case GGML_TYPE_Q6_K: return ne11 <= 7; default: @@ -418,6 +445,22 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { return ne11 <= MMVQ_MAX_BATCH_SIZE; } } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_VOLTA) { + switch (type) { + case GGML_TYPE_Q2_K: + return ne11 <= 4; + case GGML_TYPE_Q3_K: + return ne11 <= 6; + case GGML_TYPE_Q4_K: + return ne11 <= 5; + case GGML_TYPE_Q5_K: + return ne11 <= 6; + case GGML_TYPE_Q6_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } if (GGML_CUDA_CC_IS_CDNA(cc)) { if (GGML_CUDA_CC_IS_CDNA1(cc)) { switch (type) { @@ -892,6 +935,26 @@ static __global__ void mul_mat_vec_q( // x block quant index when casting the quants to int const int kqs = vdr * (tid % (qi/vdr)); +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK + // start the next iterations' weight loads early + if constexpr (mmvq_should_prefetch(type)) { + constexpr int pf_dist = 2; // loop iterations, not blocks + const int kbx_pf = kbx + pf_dist*blocks_per_iter; + if (kbx_pf < blocks_per_row_x) { +#pragma unroll + for (int i = 0; i < rows_per_cuda_block; ++i) { + const size_t off = (size_t)(kbx_offset + i*stride_row_x + kbx_pf) * ggml_cuda_type_traits<type>::bs; + mmvq_prefetch_l2((const char *) vx + off); + if constexpr (has_fusion) { + if (use_gate) { + mmvq_prefetch_l2((const char *) vgate + off); + } + } + } + } + } +#endif + #pragma unroll for (int j = 0; j < ncols_dst; ++j) { #pragma unroll diff --git a/ggml/src/ggml-cuda/reduce_rows.cuh b/ggml/src/ggml-cuda/reduce_rows.cuh index 968c47aa20a1..111fd838a7a3 100644 --- a/ggml/src/ggml-cuda/reduce_rows.cuh +++ b/ggml/src/ggml-cuda/reduce_rows.cuh @@ -1,11 +1,6 @@ #include "common.cuh" -// Row reduction kernel template - compute sum (norm=false) or mean (norm=true) -template <bool norm> -static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, const int ncols) { - const float * GGML_CUDA_RESTRICT x = x_ptr; - float * GGML_CUDA_RESTRICT dst = dst_ptr; - const int row = blockIdx.x; +static __device__ __forceinline__ float reduce_row_f32(const float * x, const int ncols) { const int col = threadIdx.x; float sum = 0.0f; @@ -17,7 +12,7 @@ static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, con for (int i = col; i < ncols;) { for (int j = 0; j < num_unroll; ++j) { if (i < ncols) { - temp[j] = x[row * ncols + i]; + temp[j] = x[i]; } else { temp[j] = 0; } @@ -35,6 +30,40 @@ static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, con __shared__ float shared_vals[32]; sum = block_reduce<block_reduce_method::SUM>(sum, shared_vals); + return sum; +} + +// Row reduction kernel template - compute sum (norm=false) or mean (norm=true) +template <bool norm> +static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, const int ncols) { + float * GGML_CUDA_RESTRICT dst = dst_ptr; + const int64_t row = blockIdx.x; + const int col = threadIdx.x; + + const float * GGML_CUDA_RESTRICT x = x_ptr + row*ncols; + const float sum = reduce_row_f32(x, ncols); + + if (col != 0) { + return; + } + + dst[row] = norm ? sum / ncols : sum; +} + +template <bool norm> +static __global__ void reduce_rows_f32_strided(const char * x_ptr, float * dst_ptr, const int ncols, + const int64_t ne1, const int64_t ne2, const int64_t nb1, const int64_t nb2, const int64_t nb3) { + float * GGML_CUDA_RESTRICT dst = dst_ptr; + const int64_t row = blockIdx.x; + const int col = threadIdx.x; + + const int64_t i1 = row % ne1; + const int64_t i2 = (row / ne1) % ne2; + const int64_t i3 = row / (ne1 * ne2); + + const float * GGML_CUDA_RESTRICT x = (const float *) (x_ptr + i1*nb1 + i2*nb2 + i3*nb3); + const float sum = reduce_row_f32(x, ncols); + if (col != 0) { return; } diff --git a/ggml/src/ggml-cuda/sumrows.cu b/ggml/src/ggml-cuda/sumrows.cu index 0003658ca95b..aa8342b5f3c7 100644 --- a/ggml/src/ggml-cuda/sumrows.cu +++ b/ggml/src/ggml-cuda/sumrows.cu @@ -24,24 +24,30 @@ void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous_rows(src0)); const int64_t ncols = src0->ne[0]; const int64_t nrows = ggml_nrows(src0); + if (ggml_is_contiguous(src0)) { + sum_rows_f32_cuda(src0_d, dst_d, ncols, nrows, stream); + return; + } + const dim3 block_nums(nrows, 1, 1); const int id = ggml_cuda_get_device(); const int nsm = ggml_cuda_info().devices[id].nsm; + dim3 block_dims; if ((nrows / nsm) < 2) { // Increase num threads to 512 for small nrows to better hide the latency - const dim3 block_dims(512, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); - ggml_cuda_kernel_launch(reduce_rows_f32</*norm=*/false>, launch_params, src0_d, dst_d, ncols); + block_dims = dim3(512, 1, 1); } else { // Enough active SMs to hide latency, use smaller blocks to allow better scheduling - const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); - ggml_cuda_kernel_launch(reduce_rows_f32</*norm=*/false>, launch_params, src0_d, dst_d, ncols); + block_dims = dim3(ncols < 1024 ? 32 : 128, 1, 1); } + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + const char * src0_d_bytes = (const char *) src0->data; + ggml_cuda_kernel_launch(reduce_rows_f32_strided</*norm=*/false>, launch_params, src0_d_bytes, dst_d, ncols, + src0->ne[1], src0->ne[2], src0->nb[1], src0->nb[2], src0->nb[3]); } diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh index ec117c57dfbc..f2a6f2009c9a 100644 --- a/ggml/src/ggml-cuda/vecdotq.cuh +++ b/ggml/src/ggml-cuda/vecdotq.cuh @@ -936,16 +936,20 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1( v[0] = q4[0]; v[1] = q4[4]; + // branchless so nvcc can hoist this out of the ncols_dst loop const uint16_t * scales = (const uint16_t *)bq4_K->scales; + const int j = bq8_offset/2; + const int jm = j & 1; + + const uint32_t s0 = scales[jm + 0]; + const uint32_t s2 = scales[jm + 2]; + const uint32_t s4 = scales[jm + 4]; + + const uint32_t hi = (uint32_t) -(int32_t) (j >= 2); + uint16_t aux[2]; - const int j = bq8_offset/2; - if (j < 2) { - aux[0] = scales[j+0] & 0x3f3f; - aux[1] = scales[j+2] & 0x3f3f; - } else { - aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2); - aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2); - } + aux[0] = (uint16_t) (((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi)); + aux[1] = (uint16_t) (((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi)); const uint8_t * sc = (const uint8_t *)aux; const uint8_t * m = sc + 2; @@ -981,16 +985,21 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1( vh[0] = qh[0] >> bq8_offset; vh[1] = qh[4] >> bq8_offset; + // same as q4_K const uint16_t * scales = (const uint16_t *)bq5_K->scales; + const int j = bq8_offset/2; + const int jm = j & 1; + + const uint32_t s0 = scales[jm + 0]; + const uint32_t s2 = scales[jm + 2]; + const uint32_t s4 = scales[jm + 4]; + + const uint32_t hi = (uint32_t) -(int32_t) (j >= 2); + uint16_t aux[2]; - const int j = bq8_offset/2; - if (j < 2) { - aux[0] = scales[j+0] & 0x3f3f; - aux[1] = scales[j+2] & 0x3f3f; - } else { - aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2); - aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2); - } + aux[0] = (uint16_t) (((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi)); + aux[1] = (uint16_t) (((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi)); + const uint8_t * sc = (const uint8_t *)aux; const uint8_t * m = sc + 2; diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index 2982eb1b14c3..2cb0d87a4d6e 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -228,9 +228,9 @@ #define __CUDA_ARCH__ 1300 -#if defined(__gfx900__) || defined(__gfx906__) +#if defined(__gfx900__) || defined(__gfx906__) || defined(__gfx909__) || defined(__gfx90c__) #define GCN5 -#endif // defined(__gfx900__) || defined(__gfx906__) +#endif // defined(__gfx900__) || defined(__gfx906__) || defined(__gfx909__) || defined(__gfx90c__) #if defined(__gfx803__) #define GCN4 @@ -325,7 +325,15 @@ static __device__ __forceinline__ int __vsubss4(const int a, const int b) { } static __device__ __forceinline__ int __vsub4(const int a, const int b) { - return __vsubss4(a, b); + // do some small modifications to a and b to make the subtraction not underflow + const unsigned int a_large = a | 0x80808080; + const unsigned int b_small = b & 0x7f7f7f7f; + const unsigned int result_low_7bits = a_large - b_small; + + // if two ops share the same high bit, we should flip the high bit of the result + const unsigned int should_flip_high_1bit = (a ^ ~b) & 0x80808080; + + return result_low_7bits ^ should_flip_high_1bit; } static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigned int b) { @@ -341,13 +349,13 @@ static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigne } static __device__ __forceinline__ unsigned int __vcmpne4(unsigned int a, unsigned int b) { - const uint8x4_t& va = reinterpret_cast<const uint8x4_t&>(a); - const uint8x4_t& vb = reinterpret_cast<const uint8x4_t&>(b); - unsigned int c; - uint8x4_t& vc = reinterpret_cast<uint8x4_t&>(c); -#pragma unroll - for (int i = 0; i < 4; ++i) { - vc[i] = va[i] == vb[i] ? 0x00 : 0xff; - } - return c; + const unsigned int x = a ^ b; + + // any non-equal bit in a byte will set the high bit of that byte here + // the addition will not overflow in the byte as op1 and op2 are both less than 0x80 + const unsigned int ne_low_7bits = ((x & 0x7f7f7f7f) + 0x7f7f7f7f) & 0x80808080; + const unsigned int ne_high_1bit = x & 0x80808080; + const unsigned int ne_any_bit = ne_low_7bits | ne_high_1bit; + + return (ne_any_bit >> 7) * 0xff; } diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 104201daff51..58806e37f7f1 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -21,6 +21,8 @@ #include <queue> #include <deque> #include <algorithm> +#include <cmath> +#include <initializer_list> #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN @@ -52,10 +54,15 @@ #include "htp-opnode.h" #include "htp-ops.h" #include "htp/matmul-ops.h" +#include "htp/binary-ops.h" #include "htp/flash-attn-ops.h" #include "htp/unary-ops.h" #include "htp/get-rows-ops.h" #include "htp/set-rows-ops.h" +#include "htp/softmax-ops.h" +#include "htp/rope-ops.h" +#include "htp/ssm-conv.h" +#include "htp/gated-delta-net-ops.h" #include "htp_iface.h" #include "htp-drv.h" @@ -65,7 +72,6 @@ using u32vec = std::vector<uint32_t>; #define GGML_HEXAGON_MAX_SESSIONS 16 -#define GGML_HEXAGON_FENCE_BUFFER_SIZE 8192 #define GGML_HEXAGON_FENCE_SLOT_SIZE 128 struct ggml_hexagon_device_config { @@ -74,6 +80,8 @@ struct ggml_hexagon_device_config { int domain_id = 0; std::string domain_name; std::string name; + + std::vector<ggml_hexagon_device_config> mdev_group; }; static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS]; @@ -88,9 +96,11 @@ static int opt_etm = 0; static int opt_verbose = 0; static int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu) static bool opt_hostbuf = false; +static bool opt_dma64 = false; -static int opt_mm_select = 3; // 3 = HMX -> Tiled -> Flat -> CPU, 2 = Tiled -> Flat -> CPU, 1 = Flat -> CPU +static int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) static int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) +static int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported) static int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (DMA, default), 1 = unfused ALLREDUCE (DMA), 0 = fallback to CPY+FENCE // Default PMU events, if profiling with PMU (mode=2) is enabled @@ -110,6 +120,7 @@ enum ggml_hexagon_fusion_flags { GGML_HEXAGON_FUSE_MUL_MAT_ADD = (1 << 3), // 8 GGML_HEXAGON_FUSE_MUL_MAT_NX = (1 << 4), // 16 GGML_HEXAGON_FUSE_MUL_MAT_ID_NX = (1 << 5), // 32 + GGML_HEXAGON_FUSE_GDN_CPY = (1 << 6), // 64 }; static inline bool ggml_hexagon_is_fusion_enabled(int flag) { @@ -172,6 +183,13 @@ static const char * htp_event_name(uint16_t id) { case HTP_TRACE_EVT_HVX_FA_Q_PREP: return "HVX_Q_PREP"; case HTP_TRACE_EVT_HVX_FA_K_PREP: return "HVX_K_PREP"; case HTP_TRACE_EVT_HVX_FA_V_PREP: return "HVX_V_PREP"; + case HTP_TRACE_EVT_HVX_GDN_PREP: return "HVX_GDN_PREP"; + case HTP_TRACE_EVT_HVX_GDN_SOLVE: return "HVX_GDN_SOLVE"; + case HTP_TRACE_EVT_HVX_GDN_V_PREP: return "HVX_GDN_V_PREP"; + case HTP_TRACE_EVT_HVX_GDN_D_PREP: return "HVX_GDN_D_PREP"; + case HTP_TRACE_EVT_HVX_GDN_OUT: return "HVX_GDN_OUT"; + case HTP_TRACE_EVT_HVX_GDN_STATE: return "HVX_GDN_STATE"; + case HTP_TRACE_EVT_HVX_GDN_REM: return "HVX_GDN_REM"; case HTP_TRACE_EVT_HMX_COMP: return "HMX_COMP"; case HTP_TRACE_EVT_L2FLUSH: return "L2FLUSH"; case HTP_TRACE_EVT_INIT: return "INIT"; @@ -248,7 +266,20 @@ enum ggml_hexagon_tensor_flags { static inline bool ggml_hexagon_is_repack_type(enum ggml_type type) { return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL || - type == GGML_TYPE_MXFP4; + type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q6_K || + type == GGML_TYPE_Q4_K; +} + +// Size of one repacked row in the DSP tiled layout. The Q6_K and Q4_K tiles store uncompressed scales/mins, +// so they are larger than the ggml blocks. For the other repack types the tile has the same size as the ggml blocks. +static inline size_t ggml_hexagon_tiled_row_size(enum ggml_type type, int64_t ne0) { + if (type == GGML_TYPE_Q6_K) { + return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q6_K / 32); + } + if (type == GGML_TYPE_Q4_K) { + return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q4_1 / 32); + } + return ggml_row_size(type, ne0); } static inline bool ggml_hexagon_is_hmx_weight_type(enum ggml_type type) { @@ -283,6 +314,15 @@ static void ggml_hexagon_precompute_unary_params( struct htp_unary_kernel_params * kparams ); +static bool ggml_hexagon_precompute_binary_params( + const struct ggml_hexagon_session * sess, + uint32_t op, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_binary_kernel_params * kparams +); + static void ggml_hexagon_precompute_get_rows_params( const struct ggml_hexagon_session * sess, const struct ggml_tensor * src0, @@ -299,6 +339,32 @@ static void ggml_hexagon_precompute_set_rows_params( struct htp_set_rows_kernel_params * kparams ); +static void ggml_hexagon_precompute_softmax_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_softmax_kernel_params * kparams +); + +static void ggml_hexagon_precompute_rope_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_rope_kernel_params * kparams +); + +static void ggml_hexagon_precompute_ssm_conv_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_ssm_conv_kernel_params * kparams +); + +static void ggml_hexagon_precompute_gated_delta_net_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_gdn_kernel_params * kparams +); + static void ggml_hexagon_precompute_fused_mmnx_params( const struct ggml_hexagon_session * sess, const struct ggml_tensor * src0, @@ -327,6 +393,7 @@ static bool ggml_hexagon_precompute_allreduce_params( ); static bool mm_is_hmx_eligible(const ggml_tensor * t); +static htp_op_code op_remap_to_htp(const ggml_tensor * t); static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams); static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams); static bool is_mergeable_mul_mat(const ggml_tensor * t); @@ -343,27 +410,48 @@ struct ggml_hexagon_tensor_extra { }; static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) { - if (!t || !t->extra) return false; + if (!t->extra) return false; auto extra = (const struct ggml_hexagon_tensor_extra *) t->extra; return (extra->flags & GGML_HEXAGON_TENSOR_FUSEABLE) != 0; } +static inline bool ggml_hexagon_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) { + const uintptr_t a0 = (uintptr_t) a->data; + const uintptr_t b0 = (uintptr_t) b->data; + const uintptr_t a1 = a0 + ggml_nbytes(a); + const uintptr_t b1 = b0 + ggml_nbytes(b); + + return a0 < b1 && b0 < a1; +} + struct htp_opnode; struct ggml_hexagon_opbatch; struct ggml_hexagon_opqueue; struct ggml_hexagon_shared_buffer; +struct ggml_hexagon_fence_buffer; struct ggml_hexagon_session; +struct ggml_backend_hexagon_device_context; + +struct ggml_hexagon_mdev_group { + uint32_t idx = 0; + uint32_t count = 1; + std::vector<std::unique_ptr<ggml_hexagon_session>> sessions; +}; struct ggml_backend_hexagon_comm_context { std::vector<ggml_backend_t> backends; size_t n_backends = 0; - uint32_t fence_seq = 0; + volatile uint32_t * fence_slots[GGML_HEXAGON_MAX_SESSIONS] = {}; + ggml_tensor fence_tensors[GGML_HEXAGON_MAX_SESSIONS] = {}; }; struct ggml_hexagon_event { - ggml_hexagon_session * sess = nullptr; - uint64_t seq = 0; + ggml_hexagon_session * sess = nullptr; + ggml_hexagon_session * fence_sess = nullptr; + volatile uint32_t * fence_slot = nullptr; + ggml_tensor fence_tensor = {}; + uint32_t seq = 0; }; struct ggml_hexagon_session { @@ -380,27 +468,35 @@ struct ggml_hexagon_session { bool valid_queue; bool valid_iface; - std::atomic<int> op_pending; ggml_hexagon_opbatch* op_batch; ggml_hexagon_opqueue* op_queue; std::unordered_map<int, std::unique_ptr<ggml_hexagon_shared_buffer>> cloned_buffers; - std::unordered_set<ggml_hexagon_session *> sync_peers; + std::unordered_set<ggml_hexagon_session *> virt_peers; + std::unordered_set<ggml_hexagon_session *> phys_peers; uint32_t n_threads = 0; uint32_t n_hvx = 0; uint32_t n_hmx = 0; uint64_t vtcm_size = 0; size_t max_vmem = 0; - size_t max_bufsize = 0; - uint32_t fence_seq; + uint32_t fence_seq = 0; + + std::atomic<uint64_t> batch_req_seq{0}; + std::atomic<uint64_t> batch_rsp_seq{0}; + std::atomic<uint32_t> last_error{HTP_STATUS_OK}; uint64_t cached_uid = 0; std::vector<htp_opnode> cached_nodes; mutable std::unordered_set<const ggml_tensor *> needs_repack; - ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr) noexcept(false); + ggml_hexagon_mdev_group mdev; + ggml_backend_dev_t dev = nullptr; + ggml_backend_hexagon_device_context * dev_ctx = nullptr; + ggml_hexagon_fence_buffer * fence_buf = nullptr; + + ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr, uint32_t mdev_idx = 0, uint32_t mdev_count = 0) noexcept(false); ~ggml_hexagon_session() noexcept(true); const char* c_name() const { return name.c_str(); } @@ -408,31 +504,38 @@ struct ggml_hexagon_session { void allocate(const ggml_hexagon_device_config & config) noexcept(false); void release() noexcept(true); + uint8_t * alloc_fence(uint32_t n_slots = 1); + void free_fence(void * ptr, uint32_t n_slots = 1); + + uint8_t * mdev_fence_slot = nullptr; + std::unordered_map<uint64_t, volatile uint32_t *> cpy_fence_slots; + + void enqueue_mdev_group(); void enqueue_op(const htp_opnode & node); void enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor = nullptr, uint32_t fence_seq = 0); - void enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq = 0); - void enqueue_allreduce(const ggml_tensor * dst, const std::vector<const ggml_tensor *> & src_tensors, const std::vector<const ggml_tensor *> & sync_tensors, uint32_t rank, uint32_t n_ranks, uint32_t fence_seq_entry = 0, uint32_t fence_seq_exit = 0); - - void flush(bool all = true); - void flush_pending(bool all = false); + void enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq = 0, bool wait = true); + void enqueue_allreduce(const ggml_tensor * dst, const std::vector<const ggml_tensor *> & src_tensors, + const std::vector<const ggml_tensor *> & sync_tensors, uint32_t rank, uint32_t n_ranks, + uint32_t fence_seq_entry = 0, uint32_t fence_seq_exit = 0); + + void start_batch(); + void flush_sync(bool all = true); + void flush_async(); void flush_batch(size_t min_ops = 1); + void flush_peers(); + void flush_pending(bool all = true); - uint64_t record_event(); - void wait_event(uint64_t seq); - + ggml_hexagon_shared_buffer * mmap_tensor(const ggml_tensor * t); bool clone_buffer(const ggml_hexagon_shared_buffer*); + void release_buffer(const ggml_hexagon_shared_buffer*); + void unclone_buffer(const ggml_hexagon_shared_buffer*); - void add_sync_peer(ggml_hexagon_session * peer) { - sync_peers.insert(peer); - } - - void flush_sync_peers() { - if (sync_peers.empty()) return; - - for (auto * peer : sync_peers) { - peer->flush_batch(); + void add_peer(ggml_hexagon_session * peer) { + if (this->phys_idx == peer->phys_idx) { + virt_peers.insert(peer); + } else { + phys_peers.insert(peer); } - sync_peers.clear(); } }; @@ -442,10 +545,10 @@ struct ggml_backend_hexagon_device_context { int dev_id; ggml_hexagon_device_config config; ggml_backend_dev_t dev = nullptr; - size_t max_bufsize = 0; - ggml_backend_buffer_type buffer_type = {}; - ggml_backend_buffer_type host_buffer_type = {}; + ggml_backend_buffer_type buffer_type = {}; + ggml_backend_buffer_type host_buffer_type = {}; + ggml_backend_buffer_type fence_buffer_type = {}; std::unique_ptr<ggml_hexagon_session> sess; @@ -477,6 +580,8 @@ struct ggml_hexagon_rpcmem_block { int fd = -1; size_t size = 0; + std::unordered_set<ggml_hexagon_session *> mapped_clones; + ggml_hexagon_rpcmem_block(size_t size) { base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size); if (!base) { @@ -501,29 +606,23 @@ struct ggml_hexagon_shared_buffer { ggml_hexagon_session * sess; std::shared_ptr<ggml_hexagon_rpcmem_block> mem; std::vector<ggml_hexagon_tensor_extra *> tensor_extra; - uint32_t fence_head = 0; - size_t fences_size = 0; bool mapped; bool pinned; + bool extended; const char * c_name() const { return sess->c_name(); } uint8_t * base() const { return mem ? mem->base : nullptr; } size_t size() const { return mem ? mem->size : 0; } int fd() const { return mem ? mem->fd : -1; } - uint8_t * alloc_fence() { - if (fences_size == 0) return nullptr; - int max_slots = fences_size / GGML_HEXAGON_FENCE_SLOT_SIZE; - uint32_t slot = (fence_head++) % max_slots; + void mmap(bool extended = false) { + if (!this->mem) return; + if (this->mapped) return; - size_t guard_offset = size() - fences_size; - uint8_t * fence_ptr = base() + guard_offset + (size_t)slot * GGML_HEXAGON_FENCE_SLOT_SIZE; - return fence_ptr; - } + GGML_ASSERT(!this->pinned || !extended); - void mmap() { - if (!this->mem) return; - fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : FASTRPC_MAP_FD_DELAYED; + this->extended = extended; + fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : (extended ? FASTRPC_MAP_FD_DELAYED_EXTENDED : FASTRPC_MAP_FD_DELAYED); int err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags); if (err != 0) { @@ -532,8 +631,8 @@ struct ggml_hexagon_shared_buffer { throw std::runtime_error("ggml-hex: fastrpc_mmap failed (see log for details)"); } - HEX_VERBOSE("ggml-hex: %s mapped buffer: base %p size %zu fd %d pinned %u\n", - sess->c_name(), (void *) base(), size(), fd(), pinned); + HEX_VERBOSE("ggml-hex: %s mapped buffer: base %p size %zu fd %d pinned %u extended %u\n", + sess->c_name(), (void *) base(), size(), fd(), pinned, extended); this->mapped = true; } @@ -563,7 +662,9 @@ struct ggml_hexagon_shared_buffer { HEX_VERBOSE("ggml-hex: %s allocated buffer: base %p size %zu fd %d pinned %d\n", sess->c_name(), (void *) base(), this->size(), fd(), (int) pinned); - mmap(); + if (this->pinned) { + mmap(); + } } void free() { @@ -574,17 +675,18 @@ struct ggml_hexagon_shared_buffer { this->mem = nullptr; } - ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, size_t size, bool pinned = false, size_t fence_size = 0) { - this->sess = sess; - this->mapped = false; - this->pinned = pinned; - this->fences_size = fence_size; + ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, size_t size, bool pinned = false) { + this->sess = sess; + this->mapped = false; + this->pinned = pinned; + this->extended = false; - // Size adjustment inside the buffer class + // Size adjustment inside the buffer class: 4K aligned data size + 4K guard page size_t guard_offset = (size + 4095) & ~4095; - size_t total_size = guard_offset; - if (fence_size > 0) { - total_size += 4096 + fence_size; + size_t total_size = guard_offset + 4096; + if (!pinned && opt_dma64) { + constexpr size_t extended_align = 2 * 1024 * 1024; + total_size = (total_size + extended_align - 1) & ~(extended_align - 1); } alloc(total_size); @@ -592,11 +694,11 @@ struct ggml_hexagon_shared_buffer { // Clone constructor for cross-session mapping ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, const ggml_hexagon_shared_buffer & other) { - this->sess = sess; - this->mem = other.mem; - this->mapped = false; - this->pinned = other.pinned; - this->fences_size = other.fences_size; + this->sess = sess; + this->mem = other.mem; + this->mapped = false; + this->pinned = other.pinned; + this->extended = other.extended; } ~ggml_hexagon_shared_buffer() { @@ -607,13 +709,63 @@ struct ggml_hexagon_shared_buffer { } }; -static ggml_hexagon_session * ggml_backend_hexagon_buffer_get_sess(ggml_backend_buffer_t buffer) { - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(buffer->context); - return sbuf->sess; +struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer { + uint32_t slot_count = 0; + uint32_t slot_head = 0; + std::vector<uint32_t> free_slots; + ggml_backend_buffer backend_buffer{}; + + ggml_hexagon_fence_buffer(ggml_hexagon_session * sess, ggml_backend_buffer_type_t buft, size_t size) + : ggml_hexagon_shared_buffer(sess, size, false /* pinned */), + slot_count(size / GGML_HEXAGON_FENCE_SLOT_SIZE), + slot_head(0) { + backend_buffer.buft = buft; + backend_buffer.context = static_cast<ggml_hexagon_shared_buffer *>(this); + backend_buffer.size = size; + mmap(false); + } + + uint8_t * alloc_slot(uint32_t n_slots = 1) { + uint8_t * ptr = nullptr; + if (n_slots == 1 && !free_slots.empty()) { + uint32_t slot = free_slots.back(); + free_slots.pop_back(); + ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE; + } else if (slot_head + n_slots <= slot_count) { + uint32_t slot = slot_head; + slot_head += n_slots; + ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE; + } + if (ptr) { + memset(ptr, 0, (size_t) n_slots * GGML_HEXAGON_FENCE_SLOT_SIZE); + } + return ptr; + } + + void free_slot(void * ptr, uint32_t n_slots = 1) { + if (!ptr) return; + uint32_t slot = ((uint8_t *) ptr - base()) / GGML_HEXAGON_FENCE_SLOT_SIZE; + for (uint32_t i = 0; i < n_slots; i++) { + free_slots.push_back(slot + i); + } + } +}; + +inline uint8_t * ggml_hexagon_session::alloc_fence(uint32_t n_slots) { + uint8_t * ptr = fence_buf->alloc_slot(n_slots); + GGML_ASSERT(ptr); + return ptr; +} + +inline void ggml_hexagon_session::free_fence(void * ptr, uint32_t n_slots) { + if (fence_buf) { + fence_buf->free_slot(ptr, n_slots); + } } static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) { auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(buffer->context); + sbuf->sess->unclone_buffer(sbuf); delete sbuf; } @@ -1219,6 +1371,377 @@ static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t offset } } +// unsigned 6-bit value (0..63) of element e of a Q6_K block, same bit layout as dequantize_row_q6_K +static inline uint8_t q6_K_get_quant(const block_q6_K * b, int e) { + const int c = e / 128; + const int w = e % 128; + const int g = w / 32; + const int l = w % 32; + const uint8_t * ql = b->ql + c * 64; + const uint8_t * qh = b->qh + c * 32; + uint8_t lo, hi; + switch (g) { + case 0: lo = ql[l] & 0xF; hi = (qh[l] >> 0) & 3; break; + case 1: lo = ql[l + 32] & 0xF; hi = (qh[l] >> 2) & 3; break; + case 2: lo = ql[l] >> 4; hi = (qh[l] >> 4) & 3; break; + default: lo = ql[l + 32] >> 4; hi = (qh[l] >> 6) & 3; break; + } + return (uint8_t) (lo | (hi << 4)); +} + +// tile layout: see HTP_MM_WEIGHT_TILE_SIZE_Q6_K in htp/matmul-ops.h +static void repack_q6_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + GGML_ASSERT(offset == 0); + + const block_q6_K * src_matrix = (const block_q6_K *) data; + int64_t ne0 = t->ne[0]; + int64_t ne1 = t->ne[1]; + int64_t ne2 = t->ne[2]; + int64_t ne3 = t->ne[3]; + int64_t ne0_padded = hex_round_up(ne0, 32); + int64_t ne1_padded = hex_round_up(ne1, 32); + + GGML_ASSERT(ne0 % QK_K == 0); + + const int n_col_tiles = ne1_padded / 32; + const int n_k_tiles = ne0_padded / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q6_K; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + + const int64_t sb_per_row = ne0 / QK_K; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + const block_q6_K * src_slice = src_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); + uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + + memset(matrix_dst, 0, matrix_size); // padding rows and the OR-ed nibbles below need zeroed tiles + + for (int64_t r = 0; r < ne1; r++) { + const int ct = (int) (r / 32); + const int row = (int) (r % 32); + const block_q6_K * src_row = src_slice + r * sb_per_row; + + for (int kt = 0; kt < n_k_tiles; kt++) { + const int kt_local = kt % 8; // k-tile within the super-block + const block_q6_K * b = &src_row[kt / 8]; + const float d = GGML_FP16_TO_FP32(b->d); + + uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; + uint8_t * lo_pl = tile; + uint8_t * hi_pl = tile + 512; + ggml_half * sc_pl = (ggml_half *) (tile + 768); + + for (int lk = 0; lk < 32; lk++) { + const uint8_t q6 = q6_K_get_quant(b, kt_local * 32 + lk); + const int g = lk >> 2; + const int pos = row * 4 + (lk & 3); + lo_pl[(g >> 1) * 128 + pos] |= (uint8_t) ((q6 & 0xF) << ((g & 1) * 4)); + hi_pl[(g >> 2) * 128 + pos] |= (uint8_t) ((q6 >> 4) << ((g & 3) * 2)); + } + for (int sub = 0; sub < 2; sub++) { + sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16(d * (float) b->scales[kt_local * 2 + sub]); + } + } + } + } + } + + GGML_UNUSED(size); +} + +// Reverse of repack_q6_K_tiled. Unpacks quants losslessly and normalizes sub-block scales. Read-back only. +static void repack_tiled_q6_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + GGML_ASSERT(offset == 0); + + block_q6_K * dst_matrix = (block_q6_K *) data; + int64_t ne0 = t->ne[0]; + int64_t ne1 = t->ne[1]; + int64_t ne2 = t->ne[2]; + int64_t ne3 = t->ne[3]; + int64_t ne0_padded = hex_round_up(ne0, 32); + int64_t ne1_padded = hex_round_up(ne1, 32); + + GGML_ASSERT(ne0 % QK_K == 0); + + const int n_col_tiles = ne1_padded / 32; + const int n_k_tiles = ne0_padded / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q6_K; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + + const int64_t sb_per_row = ne0 / QK_K; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + block_q6_K * dst_slice = dst_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); + const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + + for (int64_t r = 0; r < ne1; r++) { + const int ct = (int) (r / 32); + const int row = (int) (r % 32); + block_q6_K * dst_row = dst_slice + r * sb_per_row; + + for (int64_t sb = 0; sb < sb_per_row; sb++) { + block_q6_K * b = &dst_row[sb]; + memset(b, 0, sizeof(block_q6_K)); + + float sub_scales[16]; + for (int kt_local = 0; kt_local < 8; kt_local++) { + const int kt = sb * 8 + kt_local; + const uint8_t * tile = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; + const uint8_t * lo_pl = tile; + const uint8_t * hi_pl = tile + 512; + const ggml_half * sc_pl = (const ggml_half *) (tile + 768); + + const int c = kt_local / 4; + const int g = kt_local % 4; + uint8_t * ql = b->ql + c * 64; + uint8_t * qh = b->qh + c * 32; + + for (int lk = 0; lk < 32; lk++) { + const int g_tile = lk >> 2; + const int pos = row * 4 + (lk & 3); + const uint8_t lo = (lo_pl[(g_tile >> 1) * 128 + pos] >> ((g_tile & 1) * 4)) & 0xF; + const uint8_t hi = (hi_pl[(g_tile >> 2) * 128 + pos] >> ((g_tile & 3) * 2)) & 3; + + switch (g) { + case 0: + ql[lk] |= lo; + qh[lk] |= (hi << 0); + break; + case 1: + ql[lk + 32] |= lo; + qh[lk] |= (hi << 2); + break; + case 2: + ql[lk] |= (lo << 4); + qh[lk] |= (hi << 4); + break; + default: + ql[lk + 32] |= (lo << 4); + qh[lk] |= (hi << 6); + break; + } + } + + for (int sub = 0; sub < 2; sub++) { + sub_scales[kt_local * 2 + sub] = GGML_FP16_TO_FP32(sc_pl[sub * 32 + row]); + } + } + + float max_abs_scale = 0.0f; + for (int s = 0; s < 16; s++) { + float abs_scale = fabsf(sub_scales[s]); + if (abs_scale > max_abs_scale) { + max_abs_scale = abs_scale; + } + } + + if (max_abs_scale == 0.0f) { + b->d = GGML_FP32_TO_FP16(0.0f); + memset(b->scales, 0, sizeof(b->scales)); + } else { + float d_flt = max_abs_scale / 127.0f; + b->d = GGML_FP32_TO_FP16(d_flt); + float d_actual = GGML_FP16_TO_FP32(b->d); + float inv_d = (d_actual != 0.0f) ? (1.0f / d_actual) : 0.0f; + for (int s = 0; s < 16; s++) { + int sc = (int) roundf(sub_scales[s] * inv_d); + b->scales[s] = (int8_t) (std::max)(-128, (std::min)(127, sc)); + } + } + } + } + } + } + + GGML_UNUSED(size); +} + +static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { + if (j < 4) { + *d = q[j] & 63; + *m = q[j + 4] & 63; + } else { + *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); + *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4); + } +} + +// tile layout: see HTP_MM_WEIGHT_TILE_SIZE_Q4_1 in htp/matmul-ops.h +static void repack_q4_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + GGML_ASSERT(offset == 0); + + const block_q4_K * src_matrix = (const block_q4_K *) data; + int64_t ne0 = t->ne[0]; + int64_t ne1 = t->ne[1]; + int64_t ne2 = t->ne[2]; + int64_t ne3 = t->ne[3]; + int64_t ne0_padded = hex_round_up(ne0, 32); + int64_t ne1_padded = hex_round_up(ne1, 32); + + GGML_ASSERT(ne0 % QK_K == 0); + + const int n_col_tiles = ne1_padded / 32; + const int n_k_tiles = ne0_padded / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + + const int64_t sb_per_row = ne0 / QK_K; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + const block_q4_K * src_slice = src_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); + uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + + memset(matrix_dst, 0, matrix_size); + + for (int64_t r = 0; r < ne1; r++) { + const int ct = (int) (r / 32); + const int row = (int) (r % 32); + const block_q4_K * src_row = src_slice + r * sb_per_row; + + for (int kt = 0; kt < n_k_tiles; kt++) { + const int kt_local = kt % 8; + const block_q4_K * b = &src_row[kt / 8]; + const float d = GGML_FP16_TO_FP32(b->d); + const float dmin = GGML_FP16_TO_FP32(b->dmin); + + uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; + + uint8_t sc, m; + get_scale_min_k4(kt_local, b->scales, &sc, &m); + + const float D = d * (float) sc; + const float M = -dmin * (float) m; + + const uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; + const int shift = (kt_local & 1) ? 4 : 0; + + for (int cp = 0; cp < 16; cp++) { + const uint8_t q0 = (qs_sub[2 * cp + 0] >> shift) & 0x0F; + const uint8_t q1 = (qs_sub[2 * cp + 1] >> shift) & 0x0F; + tile_dst[cp * 32 + row] = (uint8_t) ((q1 << 4) | q0); + } + + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); + scale_dst[2 * row + 0] = GGML_FP32_TO_FP16(D); + scale_dst[2 * row + 1] = GGML_FP32_TO_FP16(M); + } + } + } + } + + GGML_UNUSED(size); +} + +// Reverse of repack_q4_K_tiled. Unpacks quants and normalizes scales/mins. Read-back only. +static void repack_tiled_q4_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + GGML_ASSERT(offset == 0); + + block_q4_K * dst_matrix = (block_q4_K *) data; + int64_t ne0 = t->ne[0]; + int64_t ne1 = t->ne[1]; + int64_t ne2 = t->ne[2]; + int64_t ne3 = t->ne[3]; + int64_t ne0_padded = hex_round_up(ne0, 32); + int64_t ne1_padded = hex_round_up(ne1, 32); + + GGML_ASSERT(ne0 % QK_K == 0); + + const int n_col_tiles = ne1_padded / 32; + const int n_k_tiles = ne0_padded / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + + const int64_t sb_per_row = ne0 / QK_K; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + block_q4_K * dst_slice = dst_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row); + const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + + for (int64_t r = 0; r < ne1; r++) { + const int ct = (int) (r / 32); + const int row = (int) (r % 32); + block_q4_K * dst_row = dst_slice + r * sb_per_row; + + for (int64_t sb = 0; sb < sb_per_row; sb++) { + block_q4_K * b = &dst_row[sb]; + memset(b, 0, sizeof(block_q4_K)); + + float sub_scales[8]; + float sub_mins[8]; + + for (int kt_local = 0; kt_local < 8; kt_local++) { + const int kt = sb * 8 + kt_local; + const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + + uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; + const int shift = (kt_local & 1) ? 4 : 0; + + for (int cp = 0; cp < 16; cp++) { + const uint8_t val = tile_src[cp * 32 + row]; + const uint8_t q0 = val & 0x0F; + const uint8_t q1 = val >> 4; + qs_sub[2 * cp + 0] |= (uint8_t) (q0 << shift); + qs_sub[2 * cp + 1] |= (uint8_t) (q1 << shift); + } + + const float D = GGML_FP16_TO_FP32(scale_src[2 * row + 0]); + const float M = GGML_FP16_TO_FP32(scale_src[2 * row + 1]); + sub_scales[kt_local] = (D > 0.0f) ? D : 0.0f; + sub_mins[kt_local] = (-M > 0.0f) ? -M : 0.0f; + } + + float max_scale = 0.0f; + float max_min = 0.0f; + for (int j = 0; j < 8; j++) { + if (sub_scales[j] > max_scale) max_scale = sub_scales[j]; + if (sub_mins[j] > max_min) max_min = sub_mins[j]; + } + + float inv_scale = 0.0f; + if (max_scale > 0.0f) { + b->d = GGML_FP32_TO_FP16(max_scale / 63.0f); + const float d_actual = GGML_FP16_TO_FP32(b->d); + inv_scale = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f; + } else { + b->d = GGML_FP32_TO_FP16(0.0f); + } + + float inv_min = 0.0f; + if (max_min > 0.0f) { + b->dmin = GGML_FP32_TO_FP16(max_min / 63.0f); + const float dmin_actual = GGML_FP16_TO_FP32(b->dmin); + inv_min = (dmin_actual > 0.0f) ? (1.0f / dmin_actual) : 0.0f; + } else { + b->dmin = GGML_FP32_TO_FP16(0.0f); + } + + for (int j = 0; j < 8; j++) { + uint8_t ls = (uint8_t) roundf(inv_scale * sub_scales[j]); + uint8_t lm = (uint8_t) roundf(inv_min * sub_mins[j]); + ls = (std::min)((uint8_t) 63, ls); + lm = (std::min)((uint8_t) 63, lm); + if (j < 4) { + b->scales[j] = ls; + b->scales[j + 4] = lm; + } else { + b->scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); + b->scales[j - 4] |= ((ls >> 4) << 6); + b->scales[j - 0] |= ((lm >> 4) << 6); + } + } + } + } + } + } + + GGML_UNUSED(size); +} + static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) { switch (tensor->type) { case GGML_TYPE_Q4_0: @@ -1229,6 +1752,10 @@ static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t repack_q4_1_tiled(tensor, data, 0, size); break; + case GGML_TYPE_Q4_K: + repack_q4_K_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q8_0: repack_q8_0_tiled(tensor, data, 0, size); break; @@ -1241,6 +1768,10 @@ static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t repack_mxfp4_tiled(tensor, data, 0, size); break; + case GGML_TYPE_Q6_K: + repack_q6_K_tiled(tensor, data, 0, size); + break; + default: break; } @@ -1319,6 +1850,12 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, repack_tiled_q4_1(data, tensor, offset, size); break; + case GGML_TYPE_Q4_K: + GGML_ASSERT(offset == 0); + GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); + repack_tiled_q4_K(data, tensor, offset, size); + break; + case GGML_TYPE_Q8_0: GGML_ASSERT(offset == 0); GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); @@ -1337,6 +1874,12 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, repack_tiled_mxfp4(data, tensor, offset, size); break; + case GGML_TYPE_Q6_K: + GGML_ASSERT(offset == 0); + GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); + repack_tiled_q6_K(data, tensor, offset, size); + break; + default: memcpy(data, (const char *) tensor->data + offset, size); break; @@ -1442,6 +1985,10 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff repack_tiled_q4_1(temp_buf.data(), tensor, offset, temp_size); break; + case GGML_TYPE_Q4_K: + repack_tiled_q4_K(temp_buf.data(), tensor, offset, temp_size); + break; + case GGML_TYPE_Q8_0: repack_tiled_q8_0(temp_buf.data(), tensor, offset, temp_size); break; @@ -1454,6 +2001,10 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff repack_tiled_mxfp4(temp_buf.data(), tensor, offset, temp_size); break; + case GGML_TYPE_Q6_K: + repack_tiled_q6_K(temp_buf.data(), tensor, offset, temp_size); + break; + default: memcpy(temp_buf.data(), (const uint8_t *) tensor->data + offset, temp_size); break; @@ -1529,8 +2080,13 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buffer_type, size_t size) { auto dev_ctx = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->dev_ctx; auto sess = dev_ctx->session(); + if (sess && sess->max_vmem && size > sess->max_vmem) { + GGML_LOG_ERROR("ggml-hex: %s buffer size %zu exceeds max_vmem %zu\n", + dev_ctx->c_name(), size, sess->max_vmem); + return nullptr; + } try { - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE); + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size); } catch (const std::exception & exc) { GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", dev_ctx->c_name(), exc.what()); @@ -1542,8 +2098,13 @@ static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buffer_type, size_t size) { auto dev_ctx = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->dev_ctx; auto sess = dev_ctx->session(); + if (sess && sess->max_vmem && size > sess->max_vmem) { + GGML_LOG_ERROR("ggml-hex: %s host buffer size %zu exceeds max_vmem %zu\n", + dev_ctx->c_name(), size, sess->max_vmem); + return nullptr; + } try { - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE); + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_host_buffer_interface, sbuf, size); } catch (const std::exception & exc) { GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", dev_ctx->c_name(), exc.what()); @@ -1562,7 +2123,7 @@ static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffe int64_t ne1 = hex_round_up(t->ne[1], 32); int64_t ne2 = t->ne[2]; int64_t ne3 = t->ne[3]; - return ggml_row_size(t->type, ne0) * ne1 * ne2 * ne3; + return ggml_hexagon_tiled_row_size(t->type, ne0) * ne1 * ne2 * ne3; } return ggml_nbytes(t); @@ -1570,8 +2131,8 @@ static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffe } static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { - auto * context = static_cast<ggml_backend_hexagon_buffer_type_context *>(buft->context); - return context->dev_ctx->max_bufsize; + return opt_mbuf; + GGML_UNUSED(buft); } static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) { @@ -1603,7 +2164,7 @@ static ggml_backend_buffer_type_i ggml_backend_hexagon_host_buffer_type_interfac }; ggml_backend_hexagon_device_context::ggml_backend_hexagon_device_context(int dev_id, const ggml_hexagon_device_config & config, ggml_backend_dev_t dev) - : dev_id(dev_id), config(config), dev(dev), max_bufsize(opt_mbuf) { + : dev_id(dev_id), config(config), dev(dev) { buffer_type.device = dev; buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name, this); @@ -1611,11 +2172,16 @@ ggml_backend_hexagon_device_context::ggml_backend_hexagon_device_context(int dev host_buffer_type.device = dev; host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface; host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-HOST", this); + + fence_buffer_type.device = dev; + fence_buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; + fence_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-FENCE", this); } ggml_backend_hexagon_device_context::~ggml_backend_hexagon_device_context() { delete static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type.context); delete static_cast<ggml_backend_hexagon_buffer_type_context *>(host_buffer_type.context); + delete static_cast<ggml_backend_hexagon_buffer_type_context *>(fence_buffer_type.context); } static bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) { @@ -1638,7 +2204,7 @@ struct ggml_hexagon_opbatch { unsigned int n_bufs; // num buffers in the batch unsigned int n_tens; // num tensors ... unsigned int n_ops; // num ops ... - size_t b_vmem; // sum of all buffer sizes + size_t b_vmem; // sum of non-extended buffer sizes unsigned int n_bufs_max; unsigned int n_tens_max; @@ -1691,17 +2257,20 @@ struct ggml_hexagon_opbatch { if (it != b_map.end()) { return it->second; } // Add new buffer to the batch - int bi = n_bufs++; GGML_ASSERT(n_bufs < HTP_OP_MAX_BUFS); + int bi = n_bufs++; b_map.insert({sbuf->fd(), bi}); htp_buf_desc &b = h_bufs[bi]; - b.base = (uint64_t) sbuf->base(); - b.fd = sbuf->fd(); - b.size = sbuf->size(); + b.base = (uint64_t) sbuf->base(); + b.fd = sbuf->fd(); + b.size = sbuf->size(); + b.flags = sbuf->extended ? HTP_BUF_EXTENDED : 0; - b_vmem += b.size; + if (!sbuf->extended) { + b_vmem += b.size; + } HEX_VERBOSE("ggml-hex: %s add-buffer #%u : fd %d base %p size %zu : vmem %zu\n", sess->c_name(), bi, b.fd, (void*) sbuf->base(), (size_t) b.size, b_vmem); @@ -1718,7 +2287,7 @@ struct ggml_hexagon_opbatch { ne0 = hex_round_up(ne0, 32); ne1 = hex_round_up(ne1, 32); } - int64_t nb1 = is_repack ? ggml_row_size(t->type, ne0) : t->nb[1]; + int64_t nb1 = is_repack ? (int64_t) ggml_hexagon_tiled_row_size(t->type, ne0) : t->nb[1]; int64_t nb2 = is_repack ? nb1 * ne1 : t->nb[2]; int64_t nb3 = is_repack ? nb2 * t->ne[2] : t->nb[3]; @@ -1767,7 +2336,7 @@ struct ggml_hexagon_opbatch { h.ne[3] = t->ne[3]; h.nb[0] = t->nb[0]; - h.nb[1] = ggml_row_size(t->type, h.ne[0]); + h.nb[1] = ggml_hexagon_tiled_row_size(t->type, h.ne[0]); h.nb[2] = h.nb[1] * h.ne[1]; h.nb[3] = h.nb[2] * h.ne[2]; h.size = h.nb[3] * h.ne[3]; @@ -1797,21 +2366,33 @@ struct ggml_hexagon_opbatch { } bool fit_op(const htp_opnode & node) const { - if (n_ops >= n_ops_max ) return false; + if (n_ops >= n_ops_max) return false; // check how much extras we will need size_t extra_bufs = 0; size_t extra_vmem = 0; size_t extra_tens = 0; + int seen_bufs[HTP_OP_MAX_BUFS]; + int n_seen_bufs = 0; + auto fit_tensor = [&](const ggml_tensor *t) { if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - if (!b_map.count(sbuf->fd())) { - extra_vmem += sbuf->size(); + int fd = sbuf->fd(); + if (!b_map.count(fd)) { + for (int i = 0; i < n_seen_bufs; i++) { + if (seen_bufs[i] == fd) return; + } + if (n_seen_bufs < HTP_OP_MAX_BUFS) { + seen_bufs[n_seen_bufs++] = fd; + } + if (!sbuf->extended) { + extra_vmem += sbuf->size(); + } extra_bufs += 1; } } @@ -1895,6 +2476,56 @@ struct ggml_hexagon_opbatch { } } + void update_mdev_group(uint32_t mdev_idx) { + if (n_ops > 0 && h_ops[0].opcode == HTP_OP_MDEV_GROUP) { + h_ops[0].params[0] = (int32_t) mdev_idx; + } + } + + bool try_fuse_common(std::initializer_list<const ggml_tensor *> tensors) const { + size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; + + int seen_bufs[HTP_OP_MAX_BUFS]; + int n_seen_bufs = 0; + + for (const auto * t : tensors) { + if (!t || t_map.count(t)) { + continue; + } + extra_tens++; + auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); + int fd = sbuf->fd(); + if (!b_map.count(fd)) { + bool found = false; + for (int i = 0; i < n_seen_bufs; i++) { + if (seen_bufs[i] == fd) { + found = true; + break; + } + } + if (!found) { + if (n_seen_bufs < HTP_OP_MAX_BUFS) { + seen_bufs[n_seen_bufs++] = fd; + } + if (!sbuf->extended) { + extra_vmem += sbuf->size(); + } + extra_bufs += 1; + } + } + } + + if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + return false; + } + + return true; + } + + bool try_fuse_common(const ggml_tensor * t1, const ggml_tensor * t2) const { + return try_fuse_common({t1, t2}); + } + bool try_fuse_allreduce_add(const htp_opnode & node) { if (n_ops == 0 || opt_ar_select != 2) return false; if (node.opcode != HTP_OP_ADD) return false; @@ -1903,15 +2534,16 @@ struct ggml_hexagon_opbatch { if (last_node.opcode != HTP_OP_ALLREDUCE) return false; auto * ar_kparams = (struct htp_allreduce_kernel_params *) last_node.kernel_params; - const uint32_t rank = (uint32_t) ar_kparams->rank; - const ggml_tensor * ar_local = (rank < last_node.inputs.size()) ? last_node.inputs[rank] : nullptr; + const uint32_t rank = (uint32_t) ar_kparams->rank; + const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; + const ggml_tensor * ar_local = last_node.inputs[rank]; const ggml_tensor * add_src0 = node.src0(); const ggml_tensor * add_src1 = node.src1(); + const ggml_tensor * add_dst = node.dst(); - if (!add_src0 || !add_src1 || !ar_local) return false; if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false; - const ggml_tensor * res_tensor = nullptr; + const ggml_tensor * res_tensor; if (add_src0 == ar_local || add_src0->data == ar_local->data) { res_tensor = add_src1; } else if (add_src1 == ar_local || add_src1->data == ar_local->data) { @@ -1920,14 +2552,12 @@ struct ggml_hexagon_opbatch { return false; } - if (!res_tensor || !res_tensor->data) return false; - if (ar_local->type != res_tensor->type) return false; const bool is_same_shape = (ar_local->ne[0] == res_tensor->ne[0] && ar_local->ne[1] == res_tensor->ne[1] && ar_local->ne[2] == res_tensor->ne[2] && ar_local->ne[3] == res_tensor->ne[3]); - const bool is_row_bcast = (ar_local->ne[0] == res_tensor->ne[0] && - res_tensor->ne[1] == 1 && res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1); + const bool is_row_bcast = !is_same_shape && (ar_local->ne[0] == res_tensor->ne[0] && res_tensor->ne[1] == 1 && + res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1); if (!is_same_shape && !is_row_bcast) return false; @@ -1940,33 +2570,27 @@ struct ggml_hexagon_opbatch { return false; } } - if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(node.dst())) { + if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(add_dst)) { return false; } + for (uint32_t r = 0; r < n_ranks; r++) { + const ggml_tensor * ar_src = last_node.inputs[r]; + if (ggml_hexagon_tensors_overlap(add_dst, ar_src)) { + HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: dst overlaps allreduce src %u\n", sess->c_name(), r); + return false; + } + } + struct htp_allreduce_kernel_params new_kparams; if (!ggml_hexagon_precompute_allreduce_params( - sess, node.dst(), (uint32_t) ar_kparams->rank, (uint32_t) ar_kparams->n_ranks, true, is_row_bcast, &new_kparams + sess, add_dst, (uint32_t) ar_kparams->rank, (uint32_t) ar_kparams->n_ranks, true, is_row_bcast, &new_kparams )) { HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", sess->c_name()); return false; } - size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; - auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; - if (!t_map.count(t)) { - extra_tens++; - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - if (!b_map.count(sbuf->fd())) { - extra_vmem += sbuf->size(); - extra_bufs += 1; - } - } - }; - fit_t(res_tensor); - fit_t(node.dst()); - if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + if (!try_fuse_common(res_tensor, add_dst)) { return false; } @@ -1974,7 +2598,7 @@ struct ggml_hexagon_opbatch { last_node.name = "ALLREDUCE+ADD"; last_node.inputs.push_back(res_tensor); last_node.outputs.clear(); - last_node.outputs.push_back(node.dst()); + last_node.outputs.push_back(add_dst); last_node.fused.push_back(node.node); memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams)); @@ -1982,9 +2606,8 @@ struct ggml_hexagon_opbatch { o.opcode = HTP_OP_ALLREDUCE_ADD; memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams)); - const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; o.src[2 * n_ranks] = add_tensor(res_tensor); - o.dst[0] = add_tensor(node.dst()); + o.dst[0] = add_tensor(add_dst); for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) { o.dst[d] = 0xffff; } @@ -2004,10 +2627,9 @@ struct ggml_hexagon_opbatch { const ggml_tensor * mul_src1 = node.src1(); const ggml_tensor * rms_out = last_node.dst(); - if (!mul_src0 || !mul_src1 || !rms_out) return false; if (!ggml_hexagon_tensor_is_fuseable(rms_out)) return false; - const ggml_tensor * weight = nullptr; + const ggml_tensor * weight; if (mul_src0 == rms_out || mul_src0->data == rms_out->data) { weight = mul_src1; } else if (mul_src1 == rms_out || mul_src1->data == rms_out->data) { @@ -2016,10 +2638,7 @@ struct ggml_hexagon_opbatch { return false; } - if (!weight || !weight->data) return false; - const ggml_tensor * src0 = last_node.src0(); - if (!src0 || !src0->data) return false; if (src0->ne[0] != weight->ne[0] || src0->ne[0] != node.dst()->ne[0]) { return false; @@ -2048,21 +2667,7 @@ struct ggml_hexagon_opbatch { return false; } - size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; - auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; - if (!t_map.count(t)) { - extra_tens++; - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - if (!b_map.count(sbuf->fd())) { - extra_vmem += sbuf->size(); - extra_bufs += 1; - } - } - }; - fit_t(weight); - fit_t(node.dst()); - if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + if (!try_fuse_common(weight, node.dst())) { return false; } @@ -2105,10 +2710,9 @@ struct ggml_hexagon_opbatch { const ggml_tensor * add_src1 = node.src1(); const ggml_tensor * mm_out = last_node.dst(); - if (!add_src0 || !add_src1 || !mm_out) return false; if (!ggml_hexagon_tensor_is_fuseable(mm_out)) return false; - const ggml_tensor * src2 = nullptr; + const ggml_tensor * src2; if (add_src0 == mm_out || add_src0->data == mm_out->data) { src2 = add_src1; } else if (add_src1 == mm_out || add_src1->data == mm_out->data) { @@ -2117,14 +2721,18 @@ struct ggml_hexagon_opbatch { return false; } - if (!src2 || !src2->data) return false; - const ggml_tensor * src0 = last_node.src0(); const ggml_tensor * src1 = last_node.src1(); - if (!src0 || !src1) return false; + if (src2->type != GGML_TYPE_F32) return false; + + const struct htp_mm_kernel_params * orig_kparams = (const struct htp_mm_kernel_params *) last_node.kernel_params; struct htp_mm_kernel_params kparams; ggml_hexagon_precompute_fused_matmul_add_params(sess, src0, src1, src2, node.dst(), &kparams); + if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED) { + return false; + } + const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; const bool can_fuse = (kparams.n_hmx > 0) || (src1_nrows == 1); if (!can_fuse) return false; @@ -2135,21 +2743,19 @@ struct ggml_hexagon_opbatch { return false; } - size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; - auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; - if (!t_map.count(t)) { - extra_tens++; - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - if (!b_map.count(sbuf->fd())) { - extra_vmem += sbuf->size(); - extra_bufs += 1; - } + if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) { + if (kparams.m_chunk < orig_kparams->m_chunk || + kparams.n_chunk < orig_kparams->n_chunk || + kparams.n_act_threads < orig_kparams->n_act_threads) { + HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, th %d->%d)\n", + sess->c_name(), orig_kparams->m_chunk, kparams.m_chunk, + orig_kparams->n_chunk, kparams.n_chunk, + orig_kparams->n_act_threads, kparams.n_act_threads); + return false; } - }; - fit_t(src2); - fit_t(node.dst()); - if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + } + + if (!try_fuse_common(src2, node.dst())) { return false; } @@ -2190,7 +2796,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * w_in = node.src0(); const ggml_tensor * x_in = node.src1(); const ggml_tensor * d_in = node.dst(); - if (!w_in || !x_in || !d_in) return false; htp_opnode & last_node = ops[n_ops - 1]; @@ -2222,21 +2827,7 @@ struct ggml_hexagon_opbatch { return false; } - size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; - auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; - if (!t_map.count(t)) { - extra_tens++; - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - if (!b_map.count(sbuf->fd())) { - extra_vmem += sbuf->size(); - extra_bufs += 1; - } - } - }; - fit_t(w_in); - fit_t(d_in); - if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + if (!try_fuse_common(w_in, d_in)) { return false; } @@ -2275,7 +2866,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * w0 = last_node.src0(); const ggml_tensor * x = last_node.src1(); const ggml_tensor * w1 = node.src0(); - if (!w0 || !x || !w1) return false; struct htp_mm_kernel_params kparams; ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, 2, &kparams); @@ -2288,21 +2878,7 @@ struct ggml_hexagon_opbatch { return false; } - size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; - auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; - if (!t_map.count(t)) { - extra_tens++; - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - if (!b_map.count(sbuf->fd())) { - extra_vmem += sbuf->size(); - extra_bufs += 1; - } - } - }; - fit_t(w1); - fit_t(node.dst()); - if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + if (!try_fuse_common(w1, node.dst())) { return false; } @@ -2352,7 +2928,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * x_in = node.src1(); const ggml_tensor * ids_in = node.node->src[2]; const ggml_tensor * d_in = node.dst(); - if (!w_in || !x_in || !ids_in || !d_in) return false; htp_opnode & last_node = ops[n_ops - 1]; @@ -2385,21 +2960,7 @@ struct ggml_hexagon_opbatch { return false; } - size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; - auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; - if (!t_map.count(t)) { - extra_tens++; - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - if (!b_map.count(sbuf->fd())) { - extra_vmem += sbuf->size(); - extra_bufs += 1; - } - } - }; - fit_t(w_in); - fit_t(d_in); - if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + if (!try_fuse_common(w_in, d_in)) { return false; } @@ -2440,7 +3001,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * x = last_node.src1(); const ggml_tensor * ids = last_node.node->src[2]; const ggml_tensor * w1 = node.src0(); - if (!w0 || !x || !ids || !w1) return false; struct htp_mm_kernel_params kparams; ggml_hexagon_precompute_fused_mmidnx_params(sess, w0, x, node.dst(), 2, &kparams); @@ -2453,21 +3013,7 @@ struct ggml_hexagon_opbatch { return false; } - size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; - auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; - if (!t_map.count(t)) { - extra_tens++; - auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); - if (!b_map.count(sbuf->fd())) { - extra_vmem += sbuf->size(); - extra_bufs += 1; - } - } - }; - fit_t(w1); - fit_t(node.dst()); - if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { + if (!try_fuse_common(w1, node.dst())) { return false; } @@ -2511,6 +3057,70 @@ struct ggml_hexagon_opbatch { return false; } + bool try_fuse_gdn_cpy(const htp_opnode & node) { + if (n_ops == 0 || node.opcode != HTP_OP_CPY) return false; + + htp_opnode & last_node = ops[n_ops - 1]; + if (last_node.opcode != HTP_OP_GATED_DELTA_NET) return false; + if (last_node.outputs.size() != 1) return false; + + const ggml_tensor * gdn_out = last_node.dst(); + const ggml_tensor * cpy_node = node.node; + const ggml_tensor * cpy_src = node.src0(); + const ggml_tensor * cpy_dst = node.dst(); + + if (!cpy_src || !cpy_dst || !cpy_dst->data) return false; + if (gdn_out->type != GGML_TYPE_F32 || cpy_src->type != GGML_TYPE_F32 || cpy_dst->type != GGML_TYPE_F32) return false; + if ((gdn_out->flags & GGML_TENSOR_FLAG_OUTPUT) || (cpy_node->flags & GGML_TENSOR_FLAG_OUTPUT)) return false; + + const ggml_tensor * v = last_node.node->src[2]; + if (!v) return false; + + const int64_t S_v = v->ne[0]; + const int64_t H = v->ne[1]; + const int64_t n_tokens = v->ne[2]; + const int64_t n_seqs = v->ne[3]; + const int64_t K = ggml_get_op_params_i32(last_node.node, 0); + const size_t tail_off = (size_t) S_v * H * n_tokens * n_seqs * sizeof(float); + + const int64_t D = S_v * S_v * H; + const int64_t n_written = std::min<int64_t>(n_tokens, K); + + if (cpy_src->op != GGML_OP_VIEW || (cpy_src->view_src != gdn_out && cpy_src->view_src->data != gdn_out->data) || + cpy_src->view_offs != tail_off || !ggml_is_contiguous(cpy_src)) { + return false; + } + + if (cpy_dst->ne[0] != D || cpy_dst->ne[1] != n_seqs || cpy_dst->nb[0] != sizeof(float)) { + return false; + } + if (n_seqs > 1 && cpy_dst->nb[1] != (size_t) D * sizeof(float)) { + return false; + } + if (n_written > 1) { + if (cpy_dst->ne[2] != n_written || cpy_dst->nb[2] != (size_t) D * n_seqs * sizeof(float)) { + return false; + } + } + + if (!try_fuse_common({cpy_dst})) { + return false; + } + + last_node.name += "+CPY"; + last_node.outputs.push_back(cpy_dst); + last_node.fused.push_back(node.node); + + htp_op_desc & o = h_ops[n_ops - 1]; + o.dst[1] = add_tensor(cpy_dst); + for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) { + o.dst[d] = 0xffff; + } + + HEX_VERBOSE("ggml-hex: %s fused GATED_DELTA_NET+CPY (#%u)\n", sess->c_name(), n_ops - 1); + return true; + } + bool try_fuse(const htp_opnode & node) { if (!opt_opfusion) return false; if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_ALLREDUCE_ADD) && try_fuse_allreduce_add(node)) return true; @@ -2518,6 +3128,7 @@ struct ggml_hexagon_opbatch { if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_ADD) && try_fuse_mul_mat_add(node)) return true; if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_NX) && try_fuse_mul_mat_nx(node)) return true; if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_ID_NX) && try_fuse_mul_mat_id_nx(node)) return true; + if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_GDN_CPY) && try_fuse_gdn_cpy(node)) return true; return false; } }; @@ -2533,17 +3144,14 @@ struct ggml_hexagon_opqueue { // Shared buffer for storing batches ggml_hexagon_shared_buffer *shm_buf; size_t shm_blk_size; - - uint64_t req_seq = 0; - uint64_t rsp_seq = 0; + size_t depth; using opvec = std::vector<htp_opnode>; - std::queue<unsigned int> done; // completed batch ids std::vector<opvec> op_cache; // per batch op cache std::vector<uint64_t> start_usec; // per batch start time - ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) { + ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) { size_t n_bufs = HTP_OP_MAX_BUFS; size_t n_ops = batch_size; size_t n_tensors = n_ops * HTP_OP_MAX_OUTPUTS + n_ops * HTP_OP_MAX_INPUTS; @@ -2564,9 +3172,6 @@ struct ggml_hexagon_opqueue { op_cache.resize(depth); start_usec.resize(depth, 0); - // init done queue - for (unsigned int i = 0; i < depth; i++) { done.push(i); } - if (opt_verbose) { GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n", sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size); @@ -2580,7 +3185,7 @@ struct ggml_hexagon_opqueue { size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; } // push new batch - bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, ggml_hexagon_opbatch* op_batch) { + bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) { static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8"); static_assert(sizeof(htp_opbatch_rsp) % 8 == 0, "sizeof(htp_opbatch_rsp) must be multiple of 8"); static_assert(sizeof(htp_buf_desc) % 8 == 0, "sizeof(htp_buf_desc) must be multiple of 8"); @@ -2588,16 +3193,17 @@ struct ggml_hexagon_opqueue { static_assert(sizeof(htp_op_desc) % 8 == 0, "sizeof(htp_op_desc) must be multiple of 8"); static_assert(sizeof(htp_prof_desc) % 8 == 0, "sizeof(htp_prof_desc) must be multiple of 8"); - if (done.empty()) { return false; } + if (seq - shm_buf->sess->batch_rsp_seq > depth) { return false; } - req.id = done.front(); done.pop(); // batch id + const uint32_t slot = (uint32_t) ((seq - 1) % depth); + + req.seq = seq; req.n_bufs = op_batch->n_bufs; req.n_tensors = op_batch->n_tens; req.n_ops = op_batch->n_ops; - req.seq = ++req_seq; - op_cache[req.id] = std::move(op_batch->ops); - start_usec[req.id] = ggml_time_us(); + op_cache[slot] = op_batch->ops; + start_usec[slot] = ggml_time_us(); const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs; const size_t t_size = sizeof(htp_tensor) * req.n_tensors; @@ -2612,7 +3218,7 @@ struct ggml_hexagon_opqueue { req.n_traces = 0; } - dbuf.ptr = shm_buf->base() + (req.id * shm_blk_size); + dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size); dbuf.fd = shm_buf->fd(); dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base(); @@ -2625,18 +3231,14 @@ struct ggml_hexagon_opqueue { uint8_t * t_ptr = m_ptr; m_ptr += t_size; uint8_t * o_ptr = m_ptr; - op_batch->sort_buffers(); - memcpy(b_ptr, (void *) op_batch->h_bufs.data(), b_size); memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size); memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size); - HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%u : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n", - shm_buf->sess->c_name(), req.id, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem, + HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%llu : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n", + shm_buf->sess->c_name(), (unsigned long long) req.seq, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem, b_size, t_size, o_size, (size_t) dbuf.size); - op_batch->reset(); - if (opt_verbose > 1) { htp_buf_desc *b = (htp_buf_desc*) b_ptr; for (unsigned int i=0; i < req.n_bufs; i++) { @@ -2645,8 +3247,8 @@ struct ggml_hexagon_opqueue { } htp_tensor *t = (htp_tensor*) t_ptr; for (unsigned int i=0; i < req.n_tensors; i++) { - GGML_LOG_DEBUG("ggml-hex: %s htp-tensor #%u : bi %u offset %u size %u : %zu:%zu:%zu:%zu\n", - shm_buf->sess->c_name(), i, t[i].bi, t[i].data, t[i].size, + GGML_LOG_DEBUG("ggml-hex: %s htp-tensor #%u : bi %u offset %llu size %u : %zu:%zu:%zu:%zu\n", + shm_buf->sess->c_name(), i, t[i].bi, (unsigned long long) t[i].data, t[i].size, (size_t) t[i].ne[0], (size_t) t[i].ne[1], (size_t) t[i].ne[2], (size_t) t[i].ne[3]); } } @@ -2655,9 +3257,7 @@ struct ggml_hexagon_opqueue { } void pop(htp_opbatch_rsp rsp, dspqueue_buffer dbuf) { - GGML_ASSERT(rsp.id < op_cache.size()); - - done.push(rsp.id); + const uint32_t slot = (uint32_t) ((rsp.seq - 1) % depth); const size_t b_size = sizeof(htp_buf_desc) * rsp.n_bufs; const size_t t_size = sizeof(htp_tensor) * rsp.n_tensors; @@ -2674,15 +3274,15 @@ struct ggml_hexagon_opqueue { const size_t m_size = b_size + t_size + o_size + p_size + tr_size; GGML_ASSERT(m_size <= shm_blk_size); - HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%u : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", - shm_buf->sess->c_name(), rsp.id, rsp.n_bufs, rsp.n_tensors, rsp.n_ops, + HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", + shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops, (size_t) dbuf.size, b_size, t_size, o_size); uint8_t * m_ptr = (uint8_t*) dbuf.ptr; uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size); if (rsp.n_ops > 0) { - auto & ops = op_cache[rsp.id]; + auto & ops = op_cache[slot]; GGML_ASSERT(rsp.n_ops <= ops.size()); const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr; @@ -2705,16 +3305,41 @@ struct ggml_hexagon_opqueue { ggml_hexagon_dump_trace_events(shm_buf->sess->name, rsp, trace_events, n_traces); } } - - if (rsp.seq > rsp_seq) { - rsp_seq = rsp.seq; - } } }; -// Flush HTP response queue i.e wait for all outstanding requests to complete +void ggml_hexagon_session::flush_peers() { + auto vpeers = std::move(virt_peers); + virt_peers.clear(); + for (auto * peer : vpeers) { + peer->flush_sync(); + } + + auto ppeers = std::move(phys_peers); + phys_peers.clear(); + for (auto * peer : ppeers) { + peer->flush_async(); + } + + for (auto & sub : this->mdev.sessions) { + sub->flush_peers(); + } +} + +void ggml_hexagon_session::flush_async() { + flush_peers(); + flush_batch(); +} + void ggml_hexagon_session::flush_pending(bool all) { - while (this->op_pending) { + for (auto & sub : this->mdev.sessions) { + sub->flush_pending(all); + if (sub->last_error > HTP_STATUS_OK) { + this->last_error = sub->last_error.load(); + } + } + + while (this->batch_rsp_seq < this->batch_req_seq) { struct htp_opbatch_rsp rsp; uint32_t rsp_size; uint32_t flags; @@ -2739,32 +3364,70 @@ void ggml_hexagon_session::flush_pending(bool all) { GGML_ABORT("ggml-hex: %s dspcall : bad response : size %u dspbufs %u\n", this->c_name(), rsp_size, n_dbufs); } - if (rsp.status != HTP_STATUS_OK) { - GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp: %s\n", this->c_name(), status_to_str(rsp.status)); - // TODO: handle errors + if (rsp.status > HTP_STATUS_OK) { + GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp %s\n", this->c_name(), status_to_str(rsp.status)); + this->last_error = rsp.status; + for (auto & sub : this->mdev.sessions) { + sub->last_error = rsp.status; + } } op_queue->pop(rsp, dbuf); - this->op_pending--; // atomic dec + GGML_ASSERT(rsp.seq == this->batch_rsp_seq + 1); + this->batch_rsp_seq = rsp.seq; if (!all) break; } } +void ggml_hexagon_session::flush_sync(bool all) { + flush_async(); + flush_pending(all); +} + +void ggml_hexagon_session::start_batch() { + if (this->mdev.count > 1) { + enqueue_mdev_group(); + } +} + void ggml_hexagon_session::flush_batch(size_t min_ops) { if (op_batch->n_ops < min_ops) { return; } + op_batch->sort_buffers(); + htp_opbatch_req req {}; dspqueue_buffer dbuf{}; - if (!op_queue->push(req, dbuf, op_batch)) { + const uint64_t seq = ++this->batch_req_seq; + + op_batch->update_mdev_group(this->mdev.idx); + + if (!op_queue->push(req, dbuf, op_batch, seq)) { flush_pending(false); - op_queue->push(req, dbuf, op_batch); + op_queue->push(req, dbuf, op_batch, seq); } - // Bump pending flag (cleared in the session::flush once we get the response) - this->op_pending++; // atomic inc + for (auto & sub : this->mdev.sessions) { + htp_opbatch_req sub_req {}; + dspqueue_buffer sub_dbuf{}; + + sub->batch_req_seq = seq; + op_batch->update_mdev_group(sub->mdev.idx); + + if (!sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq)) { + sub->flush_pending(false); + sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq); + } + + HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", sub->c_name(), sub_dbuf.ptr, sub_dbuf.size); + + int err = dspqueue_write(sub->queue, 0, 1, &sub_dbuf, sizeof(sub_req), (const uint8_t*) &sub_req, DSPQUEUE_TIMEOUT); + if (err != 0) { + GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", sub->c_name(), (unsigned) err); + } + } HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", this->c_name(), dbuf.ptr, dbuf.size); @@ -2772,28 +3435,27 @@ void ggml_hexagon_session::flush_batch(size_t min_ops) { if (err != 0) { GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", this->c_name(), (unsigned) err); } -} -void ggml_hexagon_session::flush(bool all) { - flush_sync_peers(); - flush_batch(); - flush_pending(all); + op_batch->reset(); } void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { - for (auto t : node.get_inputs()) { - if (t && t->buffer && ggml_backend_buffer_is_hexagon(t->buffer)) { - if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != this) { - this->clone_buffer(static_cast<const ggml_hexagon_shared_buffer *>(t->buffer->context)); - } + auto clone_tensor_buffer = [this](const ggml_tensor * t) { + auto sbuf = this->mmap_tensor(t); + if (!sbuf) return; + if (sbuf->sess != this) { + this->clone_buffer(sbuf); } + for (auto & sub : this->mdev.sessions) { + sub->clone_buffer(sbuf); + } + }; + + for (auto t : node.get_inputs()) { + clone_tensor_buffer(t); } for (auto t : node.get_outputs()) { - if (t && t->buffer && ggml_backend_buffer_is_hexagon(t->buffer)) { - if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != this) { - this->clone_buffer(static_cast<const ggml_hexagon_shared_buffer *>(t->buffer->context)); - } - } + clone_tensor_buffer(t); } if (opt_opfusion && op_batch->try_fuse(node)) { @@ -2801,39 +3463,86 @@ void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { } if (!op_batch->fit_op(node)) { - flush_batch(); + flush_async(); + } + + if (op_batch->empty()) { + start_batch(); + } + + if (!op_batch->fit_op(node)) { + GGML_ABORT("ggml-hex: %s op does not fit into empty batch (vmem/tensor/buffer limit exceeded)\n", + c_name()); } + op_batch->add_op(node); } +void ggml_hexagon_session::enqueue_mdev_group() { + htp_opnode group_node(HTP_OP_MDEV_GROUP); + + uint8_t * fence_slot = this->mdev_fence_slot; + + static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; + ggml_tensor dummy_t {}; + dummy_t.buffer = &this->fence_buf->backend_buffer; + dummy_t.extra = &fence_extra; + dummy_t.data = (void *) fence_slot; + dummy_t.type = GGML_TYPE_I8; + dummy_t.ne[0] = HTP_FENCE_SLOT_SIZE; + dummy_t.ne[1] = (int64_t) this->mdev.count; + dummy_t.ne[2] = 1; + dummy_t.ne[3] = 1; + dummy_t.nb[0] = 1; + dummy_t.nb[1] = HTP_FENCE_SLOT_SIZE; + dummy_t.nb[2] = dummy_t.nb[1] * dummy_t.ne[1]; + dummy_t.nb[3] = dummy_t.nb[2]; + dummy_t.op = GGML_OP_NONE; + dummy_t.op_params[0] = (int32_t) this->mdev.idx; + + ggml_tensor * node = group_node.add_dummy(dummy_t); + node->src[0] = node; + group_node.init(node); + group_node.outputs.clear(); + group_node.name = "MDEV_GROUP"; + + for (auto & sub : this->mdev.sessions) { + sub->clone_buffer(this->fence_buf); + } + + op_batch->add_op(group_node); +} + void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) { - htp_opnode cpy_node(HTP_OP_CPY); + const bool with_fence = sync_tensor != nullptr; + htp_opnode cpy_node(with_fence ? HTP_OP_CPY_FENCE : HTP_OP_CPY); ggml_tensor* node = cpy_node.add_dummy(*dst); node->op = GGML_OP_CPY; node->src[0] = const_cast<ggml_tensor *>(src); - node->src[1] = sync_tensor ? cpy_node.add_dummy(*sync_tensor) : nullptr; - if (sync_tensor) { + node->src[1] = with_fence ? cpy_node.add_dummy(*sync_tensor) : nullptr; + if (with_fence) { node->op_params[0] = (int32_t) fence_seq; } cpy_node.init(node); - if (sync_tensor) { + if (with_fence) { cpy_node.name = "CPY+FENCE"; } this->enqueue_op(cpy_node); } -void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq) { +void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq, bool wait) { htp_opnode sync_node(HTP_OP_FENCE); ggml_tensor* node = sync_node.add_dummy(*sync_tensor); node->op = GGML_OP_NONE; node->src[0] = node; node->op_params[0] = (int32_t) fence_seq; + node->op_params[1] = wait ? 0 : 1; sync_node.init(node); - sync_node.name = "FENCE"; + sync_node.name = wait ? "FENCE_WAIT" : "FENCE_SIGNAL"; this->enqueue_op(sync_node); } @@ -2851,7 +3560,6 @@ static bool ggml_hexagon_precompute_allreduce_params( kparams->n_ranks = (int32_t) n_ranks; kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0; - const uint32_t n_bufs = n_ranks + 1 + (has_add ? 1 : 0); const uint32_t nelem = (uint32_t) ggml_nelements(dst); const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float); const bool is_contiguous = ggml_is_contiguous(dst); @@ -2895,6 +3603,7 @@ static bool ggml_hexagon_precompute_allreduce_params( const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem; const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nelem / 128)); kparams->n_threads = n_threads; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); uint32_t block_elems = 65536; if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) { @@ -2904,15 +3613,15 @@ static bool ggml_hexagon_precompute_allreduce_params( kparams->block_elems = block_elems; kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_elems > 128) { - const size_t max_bytes_per_buf = sess->vtcm_size / (n_threads * n_bufs * 2); + const size_t max_bytes_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2); block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128); if (block_elems < 128) break; kparams->block_elems = block_elems; kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; } if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) { @@ -2928,6 +3637,7 @@ static bool ggml_hexagon_precompute_allreduce_params( const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem; const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nrows)); kparams->n_threads = n_threads; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); const uint32_t row_bytes = ne0 * elem_size; const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128); @@ -2939,14 +3649,14 @@ static bool ggml_hexagon_precompute_allreduce_params( kparams->block_elems = block_rows; kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_rows > 1) { - const size_t max_rows_per_buf = sess->vtcm_size / (n_threads * n_bufs * 2 * row_size_aligned); + const size_t max_rows_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned); block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf); kparams->block_elems = block_rows; kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; if (max_rows_per_buf == 0) break; } @@ -3002,41 +3712,75 @@ void ggml_hexagon_session::enqueue_allreduce( this->enqueue_op(ar_node); } -void ggml_hexagon_session::wait_event(uint64_t seq) { - flush_sync_peers(); - HEX_VERBOSE("ggml-hex: %s opqueue-wait start: seq %llu, current rsp-seq %llu, pending %d\n", - this->name.c_str(), (unsigned long long)seq, (unsigned long long)op_queue->rsp_seq, (int)this->op_pending); - while (op_queue->rsp_seq < seq && this->op_pending > 0) { - this->flush_pending(false); - } - HEX_VERBOSE("ggml-hex: %s opqueue-wait end: seq %llu, current rsp-seq %llu, pending %d\n", - this->name.c_str(), (unsigned long long)seq, (unsigned long long)op_queue->rsp_seq, (int)this->op_pending); -} +ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) { + if (!t) return nullptr; -uint64_t ggml_hexagon_session::record_event() { - flush_batch(); - return op_queue->req_seq; + auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context); + if (!sbuf->mapped) { + const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; + const bool extended = opt_dma64 && is_weight; + sbuf->mmap(extended); + } + return sbuf; } -bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer *sbuf) +bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf) { - if (this->cloned_buffers.find(sbuf->fd()) != this->cloned_buffers.end()) return true; + GGML_ASSERT(sbuf && sbuf->mem); + if (sbuf->sess == this) return true; + + auto mem = sbuf->mem; + int fd = mem->fd; + + GGML_ASSERT(fd >= 0); + + if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true; + + GGML_ASSERT(sbuf->mapped); HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(), - sbuf->c_name(), sbuf->base(), sbuf->size(), sbuf->fd()); + sbuf->c_name(), sbuf->base(), sbuf->size(), fd); auto clone = std::make_unique<ggml_hexagon_shared_buffer>(this, *sbuf); try { - clone->mmap(); + clone->mmap(sbuf->extended); } catch (const std::exception & exc) { GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what()); return false; } - this->cloned_buffers[sbuf->fd()] = std::move(clone); + this->cloned_buffers[fd] = std::move(clone); + mem->mapped_clones.insert(this); return true; } +void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) { + GGML_ASSERT(sbuf && sbuf->mem); + + auto mem = sbuf->mem; + int fd = mem->fd; + + GGML_ASSERT(fd >= 0); + + auto it = this->cloned_buffers.find(fd); + if (it != this->cloned_buffers.end()) { + auto clone = std::move(it->second); + this->cloned_buffers.erase(it); + } + mem->mapped_clones.erase(this); +} + +void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) { + GGML_ASSERT(sbuf && sbuf->mem); + + auto mem = sbuf->mem; + std::vector<ggml_hexagon_session *> sessions(mem->mapped_clones.begin(), mem->mapped_clones.end()); + + for (auto * sess : sessions) { + sess->release_buffer(sbuf); + } +} + static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) { // Allocate a bunch pinned buffers till failure. // This is kind of expensive but handy for figuring out exactly how much we can mmap on a specific device. @@ -3075,14 +3819,16 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->valid_queue = false; this->valid_iface = false; - this->phys_idx = phys_idx; - this->virt_idx = virt_idx; - this->domain_id = config.domain_id; - this->session_id = 0; - this->name = config.name; - this->op_pending = 0; + this->name = config.name; + this->phys_idx = phys_idx; + this->virt_idx = virt_idx; + this->domain_id = config.domain_id; + this->session_id = 0; + this->batch_req_seq = 0; + this->batch_rsp_seq = 0; + this->last_error = HTP_STATUS_OK; - GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str()); + GGML_LOG_DEBUG("ggml-hex: %s allocating new session : domain %u phys-idx %u virt-idx %u\n", this->name.c_str(), this->domain_id, phys_idx, virt_idx); if (config.domain_id < 0 || config.domain_name.empty()) { GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx); @@ -3091,25 +3837,14 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n const std::string & dom_name = config.domain_name; - // Enable Unsigned PD for all domains - { - struct remote_rpc_control_unsigned_module u; - u.domain = -1; - u.enable = 1; - int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); - if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err); - throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); - } - } - // Create new session if virtual_idx > 0 if (virt_idx > 0) { - struct remote_rpc_reserve_new_session n; + struct remote_rpc_reserve_new_session n {}; n.domain_name_len = dom_name.size(); n.domain_name = const_cast<char *>(dom_name.c_str()); n.session_name = const_cast<char *>(this->name.c_str()); n.session_name_len = this->name.size(); + n.session_id = virt_idx; int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n)); if (err != AEE_SUCCESS) { @@ -3123,7 +3858,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->domain_id = n.effective_domain_id; this->valid_session = true; } else { - struct remote_rpc_effective_domain_id eff = {}; + struct remote_rpc_effective_domain_id eff {}; eff.domain_name = const_cast<char *>(dom_name.c_str()); eff.domain_name_len = dom_name.size(); eff.session_id = 0; @@ -3137,6 +3872,18 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n } } + // Enable unsigned modules + { + struct remote_rpc_control_unsigned_module u; + u.domain = this->domain_id; + u.enable = 1; + int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); + if (err != AEE_SUCCESS) { + GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err); + throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); + } + } + char session_uri[256]; { char htp_uri[256]; @@ -3164,14 +3911,13 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n // Open session int err = htp_iface_open(session_uri, &this->handle); if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: %s failed to open session : error 0x%x\n", this->c_name(), err); + GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri, err); throw std::runtime_error("ggml-hex: failed to open session (see log for details)"); } this->valid_handle = true; // Query HW info and resolve session options - this->max_bufsize = opt_mbuf; { unsigned int hw_n_threads = 0; unsigned int hw_n_hvx = 0; @@ -3179,8 +3925,9 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n unsigned long long hw_vtcm_size = 0; int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size); if (hw_err == 0) { - this->n_threads = opt_nhvx > 0 ? (uint32_t)opt_nhvx : (uint32_t)hw_n_threads; - this->n_hvx = opt_nhvx > 0 ? (uint32_t)opt_nhvx : (uint32_t)hw_n_hvx; + const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads); + this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads; + this->n_hvx = this->n_threads; this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0; this->vtcm_size = (uint64_t)hw_vtcm_size; GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n", @@ -3188,8 +3935,9 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n (unsigned long long)(this->vtcm_size / (1024 * 1024))); } else { GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err); - this->n_threads = opt_nhvx > 0 ? (uint32_t)opt_nhvx : 8; - this->n_hvx = opt_nhvx > 0 ? (uint32_t)opt_nhvx : 8; + const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS); + this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads; + this->n_hvx = this->n_threads; this->n_hmx = (opt_nhmx != 0) ? 1 : 0; this->vtcm_size = 8 * 1024 * 1024; } @@ -3245,6 +3993,11 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n // Allocate buffers and state for op batching this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue); + this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024); + if (this->mdev.count > 1) { + this->mdev_fence_slot = this->alloc_fence(this->mdev.count); + } + if (!opt_vmem) { opt_vmem = ggml_hexagon_measure_max_vmem(this); GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem); @@ -3255,7 +4008,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem); // Start dspqueue/opbatch processing - err = htp_iface_start(this->handle, this->session_id, this->queue_id, opt_nhvx, opt_nhmx, this->max_vmem); + err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem); if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err); throw std::runtime_error("ggml-hex: iface start failed (see log for details)"); @@ -3276,6 +4029,8 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n void ggml_hexagon_session::release() noexcept(true) { GGML_LOG_INFO("ggml-hex: releasing session: %s\n", this->name.c_str()); + this->mdev.sessions.clear(); + int err; if (this->valid_iface) { @@ -3288,6 +4043,19 @@ void ggml_hexagon_session::release() noexcept(true) { delete this->op_batch; delete this->op_queue; + for (auto & it : this->cpy_fence_slots) { + free_fence((void *) it.second, 1); + } + this->cpy_fence_slots.clear(); + + if (this->fence_buf) { + unclone_buffer(this->fence_buf); + delete this->fence_buf; + this->fence_buf = nullptr; + } + while (!this->cloned_buffers.empty()) { + release_buffer(this->cloned_buffers.begin()->second.get()); + } if (opt_etm) { err = htp_iface_etm(this->handle, 0); @@ -3314,23 +4082,30 @@ void ggml_hexagon_session::release() noexcept(true) { if (this->valid_handle) { htp_iface_close(this->handle); } - - this->cloned_buffers.clear(); } -ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev) noexcept(false) { - op_batch = nullptr; - op_queue = nullptr; - fence_seq = ((uintptr_t)this) & 0xFFFF; +ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev, uint32_t mdev_idx, uint32_t mdev_count) noexcept(false) { + this->dev = dev; + this->dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context); + this->mdev.idx = mdev_idx; + this->mdev.count = mdev_count > 0 ? mdev_count : (uint32_t) (1 + config.mdev_group.size()); + op_batch = nullptr; + op_queue = nullptr; + fence_buf = nullptr; + fence_seq = ((uintptr_t)this) & 0xFFFF; try { allocate(config); + if (this->mdev.idx == 0 && !config.mdev_group.empty()) { + for (size_t i = 0; i < config.mdev_group.size(); i++) { + this->mdev.sessions.push_back(std::make_unique<ggml_hexagon_session>( + config.mdev_group[i], this->dev, (uint32_t) (i + 1), this->mdev.count)); + } + } } catch (const std::exception & exc) { release(); throw; } - - GGML_UNUSED(dev); } ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) { @@ -3362,7 +4137,9 @@ static bool ggml_hexagon_flash_attn_is_hmx_eligible( const uint32_t DK = q->ne[0]; const uint32_t DV = v->ne[0]; - if (DK % 64 != 0 || DV % 64 != 0) { + // Head dims that are not multiples of 64 are handled by internally padding to + // DK_pad/DV_pad = round_up(.,64) and zero-filling the tail lanes. + if (DK % 8 != 0 || DV % 8 != 0) { return false; } @@ -3435,8 +4212,13 @@ static bool ggml_hexagon_precompute_flash_attn_params( // Check HMX eligibility const struct ggml_tensor * sinks = op->src[4]; if (ggml_hexagon_flash_attn_is_hmx_eligible(sess, q, k, v, sinks)) { + // HMX tiles head_dim in units of 64; when DK/DV are not 64-aligned the kernel + // operates on padded dims with zero-filled tail lanes. VTCM budget and chunk-size + // are sized for the padded tiles. + const uint32_t DK_pad = hex_round_up(DK, 64); + const uint32_t DV_pad = hex_round_up(DV, 64); size_t Br = 0, Bc = 0; - int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK, DV, neq1, nek1, sess->vtcm_size, sess->n_threads, kparams->is_q_fp32 != 0); + int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, sess->vtcm_size, sess->n_threads, kparams->is_q_fp32 != 0, sinks != nullptr, n_head); if (ret == 0) { kparams->kernel_type = HTP_FA_KERNEL_HMX; kparams->Br = Br; @@ -3446,7 +4228,7 @@ static bool ggml_hexagon_precompute_flash_attn_params( kparams->u.hmx.g_br = hex_align_up(G * Br, 32); kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && sess->n_threads >= 2) ? 1 : 0; - kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK, DV, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, kparams->is_q_fp32 != 0); + kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, kparams->is_q_fp32 != 0, sinks != nullptr, n_head); const size_t row_vec_bytes = hex_align_up(Bc * sizeof(uint16_t), 256); kparams->u.hmx.row_buf_stride = row_vec_bytes / 128; // HVX vector is 128 bytes @@ -3477,7 +4259,7 @@ static bool ggml_hexagon_precompute_flash_attn_params( const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128); const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128); - kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, kparams->is_q_fp32 != 0, mask != nullptr, sess->n_threads); + kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, kparams->is_q_fp32 != 0, mask != nullptr, sinks != nullptr, n_head, sess->n_threads); kparams->u.hvx.size_q_row_padded = size_q_row_padded; kparams->u.hvx.size_k_row_padded = size_k_row_padded; @@ -3548,6 +4330,10 @@ static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_sess } static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { + if (opt_gdn_select < 1) { + return false; + } + const struct ggml_tensor * q = op->src[0]; const struct ggml_tensor * k = op->src[1]; const struct ggml_tensor * v = op->src[2]; @@ -3556,10 +4342,6 @@ static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_ses const struct ggml_tensor * state = op->src[5]; const struct ggml_tensor * dst = op; - if (!q || !k || !v || !g || !beta || !state) { - return false; - } - if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 || g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { @@ -3597,9 +4379,31 @@ static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_ses return false; } - return true; + const uint32_t total_rows = (uint32_t) (H * n_seqs); + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, total_rows); - GGML_UNUSED(sess); + const bool can_use_hmx = (opt_gdn_select >= 2) && + (sess->n_hmx > 0) && + (S_v % 64 == 0) && + (n_tokens >= HTP_GDN_MIN_TOKENS) && + (g->ne[0] == 1) && + (K == 1); + + if (can_use_hmx) { + struct htp_gdn_hmx_vtcm_layout layout; + uint32_t n_heads_batch = 0; + if (!htp_gdn_hmx_solve_layout(&layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, sess->vtcm_size, n_threads, true, &n_heads_batch)) { + return false; + } + } else { + struct htp_gdn_vtcm_layout layout; + htp_gdn_vtcm_layout_build(&layout, (uint32_t) S_v, n_threads); + if (layout.total_bytes > sess->vtcm_size) { + return false; + } + } + + return true; } static bool ggml_hexagon_matmul_is_hmx_eligible( @@ -3670,6 +4474,7 @@ static bool ggml_hexagon_precompute_hmx_mm_params( int ne11_padded, bool is_matmul_id, bool is_batched, + size_t src2_size, size_t vtcm_budget, struct htp_mm_kernel_params * kparams ) { @@ -3690,7 +4495,7 @@ static bool ggml_hexagon_precompute_hmx_mm_params( if (is_batched_val && wtype == GGML_TYPE_F16 && group_size > 1) { // Try grouped path first const bool use_dma_activation = (src1->nb[1]/sizeof(float) > (size_t)ne00_padded); - if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, ne11, group_size, use_dma_activation, n_threads, pipeline, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { + if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, ne11, group_size, use_dma_activation, n_threads, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { use_grouped = true; } } @@ -3698,7 +4503,7 @@ static bool ggml_hexagon_precompute_hmx_mm_params( if (!use_grouped) { // Fallback to simple 2D path (group_size = 1) const int m_id_rows = (dst && is_matmul_id) ? (int) ((size_t) dst->ne[1] * dst->ne[2]) : 0; - if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, ne11_padded, ne11, n_threads, pipeline, is_matmul_id, aligned_tile_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { + if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, ne11_padded, ne11, n_threads, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { return false; } } @@ -3711,12 +4516,13 @@ static bool ggml_hexagon_precompute_hmx_mm_params( kparams->n_act_threads = act_threads_selected; kparams->tile_size = htp_mm_get_weight_tile_size(wtype); kparams->aligned_tile_size = aligned_tile_size; - kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); kparams->vtcm_size = vtcm_size; kparams->vtcm_src0_size = 0; kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected); kparams->div_ne00_padded = init_fastdiv_values(ne00_padded); kparams->vtcm_src1_size = 0; + kparams->vtcm_src2_size = (int32_t) src2_size; kparams->vtcm_dst_size = 0; if (is_batched && !is_matmul_id) { @@ -3746,7 +4552,13 @@ static void ggml_hexagon_precompute_hvx_mm_params( size_t vtcm_budget, struct htp_mm_kernel_params * kparams ) { + if (opt_mm_select < 1) { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; + } + kparams->n_hmx = 0; + kparams->n_threads = sess->n_threads; const bool is_quant = (wtype != GGML_TYPE_F16 && wtype != GGML_TYPE_F32); const int src1_nrows = ne11 * ne12 * ne13; @@ -3760,7 +4572,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( if (is_matmul_id) { kparams->kernel_type = (src1_nrows < (int) sess->n_threads) ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; - kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); struct htp_mm_hvx_vtcm_layout L; uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; @@ -3768,29 +4580,30 @@ static void ggml_hexagon_precompute_hvx_mm_params( for (uint32_t d = max_prefetch; d >= 2; d /= 2) { htp_mm_hvx_vtcm_layout_build( &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0->nb[1], 0, src2_row_size, d, true, false + 0, src0->nb[1], kparams->src1_row_size, 0, d, true, false ); if (L.total_bytes <= vtcm_budget) { best_n_prefetch = d; break; } } - if (best_n_prefetch == 2 && L.total_bytes > vtcm_budget) { - htp_mm_hvx_vtcm_layout_build( - &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0->nb[1], 0, src2_row_size, 2, true, false - ); + if (L.total_bytes > vtcm_budget) { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; } - kparams->n_prefetch = best_n_prefetch; + kparams->n_prefetch = best_n_prefetch; kparams->vtcm_size = L.total_bytes; kparams->vtcm_src0_size = L.src0_bytes; kparams->vtcm_src1_size = L.src1_bytes; kparams->vtcm_dst_size = L.dst_bytes; + goto done_quant; } else { - bool try_tiled = (k_align && opt_mm_select >= 2); + bool try_tiled = (k_align && opt_mm_select >= 1); if (try_tiled) { - kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - if (src1_nrows < (int)sess->n_threads) { + kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) + ? htp_mm_q8_1_tiled_row_size(ne10) + : htp_mm_q8_0_tiled_row_size(ne10); + if (src1_nrows < (int) sess->n_threads) { kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_BLOCK; } else { kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; @@ -3809,113 +4622,72 @@ static void ggml_hexagon_precompute_hvx_mm_params( break; } } - if (best_n_prefetch == 2 && L.total_bytes > vtcm_budget) { - htp_mm_hvx_vtcm_layout_build( - &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 2, false, false - ); - } - kparams->n_prefetch = best_n_prefetch; - - if (L.total_bytes <= vtcm_budget) { - kparams->vtcm_size = L.total_bytes; + uint32_t m_chunk = 0; + if (htp_mm_hvx_solve_vtcm_params( + kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, best_n_prefetch, vtcm_budget, + &L, &m_chunk)) { + kparams->n_prefetch = best_n_prefetch; + kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0; + kparams->vtcm_size = L.total_bytes; kparams->vtcm_src0_size = L.src0_bytes; kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_dst_size = L.dst_bytes; + kparams->vtcm_src2_size = L.src2_bytes; + kparams->vtcm_dst_size = L.dst_bytes; goto done_quant; } - HEX_VERBOSE("ggml-hex: %s HVX tiled path VTCM size needed (%zu) > budget (%zu), falling back to HVX flat\n", sess->name.c_str(), L.total_bytes, vtcm_budget); } - // Flat HVX fallback - { - kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; - - struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build( - &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false - ); - - kparams->n_prefetch = 16; - kparams->vtcm_size = L.total_bytes; - kparams->vtcm_src0_size = L.src0_bytes; - kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_dst_size = L.dst_bytes; - } + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; } done_quant:; } else if (wtype == GGML_TYPE_F16) { // F16 HVX - const bool is_batched = (ne02 > 1) || (ne03 > 1); - const bool is_permuted = ggml_is_permuted(src0) || ggml_is_permuted(src1); - struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_F16_F16_VTCM, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false - ); - - if (!is_batched && !is_permuted && L.total_bytes <= vtcm_budget) { + uint32_t m_chunk = 0; + if (htp_mm_hvx_solve_vtcm_params( + HTP_MM_KERNEL_HVX_F16_F16_VTCM, wtype, ne10, src1_nrows, sess->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget, + &L, &m_chunk)) { kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F16_VTCM; + kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0; kparams->src1_row_size = hex_round_up(ne10 * 2, 128); kparams->vtcm_size = L.total_bytes; kparams->vtcm_src0_size = L.src0_bytes; kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_src2_size = L.src2_bytes; kparams->vtcm_dst_size = L.dst_bytes; kparams->n_prefetch = 16; - } else { - if (src1->type == GGML_TYPE_F32) { - kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F32_DDR; - } else { - kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F16_DDR; - } - kparams->src1_row_size = src1->nb[1]; - htp_mm_hvx_vtcm_layout_build( - &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false - ); - kparams->vtcm_size = L.total_bytes; - kparams->vtcm_src0_size = L.src0_bytes; - kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_dst_size = L.dst_bytes; - kparams->n_prefetch = 16; + return; } + + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; } else { // F32 HVX - const bool is_batched = (ne02 > 1) || (ne03 > 1); - const bool is_permuted = ggml_is_permuted(src0) || ggml_is_permuted(src1); - struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_F32_F32_VTCM, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false - ); - - if (!is_batched && !is_permuted && L.total_bytes <= vtcm_budget) { + uint32_t m_chunk = 0; + if (htp_mm_hvx_solve_vtcm_params( + HTP_MM_KERNEL_HVX_F32_F32_VTCM, wtype, ne10, src1_nrows, sess->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget, + &L, &m_chunk)) { kparams->kernel_type = HTP_MM_KERNEL_HVX_F32_F32_VTCM; + kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0; kparams->src1_row_size = hex_round_up(ne10 * 4, 128); kparams->vtcm_size = L.total_bytes; kparams->vtcm_src0_size = L.src0_bytes; kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_src2_size = L.src2_bytes; kparams->vtcm_dst_size = L.dst_bytes; kparams->n_prefetch = 16; - } else { - kparams->kernel_type = HTP_MM_KERNEL_HVX_F32_F32_DDR; - kparams->src1_row_size = src1->nb[1]; - htp_mm_hvx_vtcm_layout_build( - &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false - ); - kparams->vtcm_size = L.total_bytes; - kparams->vtcm_src0_size = L.src0_bytes; - kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_dst_size = L.dst_bytes; - kparams->n_prefetch = 16; + return; } + + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; } } @@ -3925,6 +4697,7 @@ static void ggml_hexagon_precompute_matmul_params_impl( const struct ggml_tensor * src1, const struct ggml_tensor * dst, const size_t src2_row_size, + const size_t src2_size, struct htp_mm_kernel_params * kparams ) { memset(kparams, 0, sizeof(*kparams)); @@ -3951,9 +4724,9 @@ static void ggml_hexagon_precompute_matmul_params_impl( const size_t vtcm_budget = sess->vtcm_size; // Check HMX eligibility and try precomputing HMX parameters - bool hmx_enabled = (sess->n_hmx > 0) && (opt_mm_select >= 3); + bool hmx_enabled = (sess->n_hmx > 0) && (opt_mm_select >= 2); if (hmx_enabled && ggml_hexagon_matmul_is_hmx_eligible(src0, src1, dst, ne01_padded, is_matmul_id, is_batched)) { - if (ggml_hexagon_precompute_hmx_mm_params(sess, src0, src1, dst, wtype, ne00_padded, ne01_padded, ne02, ne11, ne12, ne11_padded, is_matmul_id, is_batched, vtcm_budget, kparams)) { + if (ggml_hexagon_precompute_hmx_mm_params(sess, src0, src1, dst, wtype, ne00_padded, ne01_padded, ne02, ne11, ne12, ne11_padded, is_matmul_id, is_batched, src2_size, vtcm_budget, kparams)) { goto finalize; } } @@ -3966,7 +4739,7 @@ static void ggml_hexagon_precompute_matmul_params_impl( kparams->div_ne1 = init_fastdiv_values(ne11); kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1); kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1); - kparams->div_ne11 = init_fastdiv_values(ne11); + kparams->div_ne12 = init_fastdiv_values(ne12); } static void ggml_hexagon_precompute_matmul_params( @@ -3976,7 +4749,7 @@ static void ggml_hexagon_precompute_matmul_params( const struct ggml_tensor * dst, struct htp_mm_kernel_params * kparams ) { - ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, 0, kparams); + ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, 0, 0, kparams); } static void ggml_hexagon_precompute_fused_matmul_add_params( @@ -3987,7 +4760,83 @@ static void ggml_hexagon_precompute_fused_matmul_add_params( const struct ggml_tensor * dst, struct htp_mm_kernel_params * kparams ) { - ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, src2->nb[1], kparams); + const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0; + ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, kparams); +} + +static bool ggml_hexagon_precompute_binary_params( + const struct ggml_hexagon_session * sess, + uint32_t op, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_binary_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const size_t elem_size = ggml_type_size(src0->type); + const size_t src0_row_size = src0->ne[0] * elem_size; + const size_t src1_row_size = src1->ne[0] * elem_size; + const size_t dst_row_size = dst->ne[0] * elem_size; + + const size_t src0_row_size_aligned = hex_round_up(src0_row_size, 128); + const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128); + const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128); + + const bool is_add_id = op == HTP_OP_ADD_ID; + const bool is_scalar = !is_add_id && src1->ne[0] == 1; + const bool is_transposed = src0->nb[1] < src0_row_size || src1->nb[1] < src1_row_size || dst->nb[1] < dst_row_size; + const bool is_same_shape = !is_add_id && !is_scalar && !is_transposed && + src1->ne[0] == src0->ne[0] && + (src1->ne[1] == src0->ne[1] || src1->ne[1] == 1) && + (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) && + (src1->ne[3] == src0->ne[3] || src1->ne[3] == 1); + const bool is_row_bcast = is_same_shape && src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1; + const bool is_complex = !is_add_id && !is_scalar && !is_same_shape && (src1->ne[0] == src0->ne[0]); + + enum htp_binary_kernel_type kernel_type; + size_t src1_size = 0; + + if (is_add_id) { + kernel_type = HTP_BINARY_KERNEL_ADD_ID; + src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128); + } else if (is_row_bcast) { + kernel_type = HTP_BINARY_KERNEL_ROW_BCAST; + src1_size = src1_row_size_aligned; + } else if (is_scalar) { + const bool is_scalar_static = (src1->ne[2] == 1 && src1->ne[3] == 1) && + (src1->ne[1] == 1 || src1->nb[1] == elem_size); + if (is_scalar_static) { + kernel_type = HTP_BINARY_KERNEL_SCALAR_DMA; + src1_size = hex_round_up(src1->ne[1] * elem_size, 128); + } else { + kernel_type = HTP_BINARY_KERNEL_SCALAR; + } + } else if (is_same_shape) { + kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE; + } else if (is_complex) { + kernel_type = HTP_BINARY_KERNEL_COMPLEX; + } else { + kernel_type = HTP_BINARY_KERNEL_REPEAT; + } + + kparams->kernel_type = kernel_type; + kparams->n_threads = sess->n_threads; + kparams->src0_row_size_aligned = src0_row_size_aligned; + kparams->src1_row_size_aligned = src1_row_size_aligned; + kparams->dst_row_size_aligned = dst_row_size_aligned; + kparams->src1_size = src1_size; + + struct htp_binary_vtcm_layout L; + htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size); + if (L.rows_per_buffer == 0 || L.total_bytes > sess->vtcm_size) { + return false; + } + + kparams->rows_per_buffer = L.rows_per_buffer; + kparams->vtcm_size = L.total_bytes; + + return true; } static void ggml_hexagon_precompute_unary_params( @@ -4148,6 +4997,276 @@ static void ggml_hexagon_precompute_set_rows_params( kparams->vtcm_size = vtcm_layout.total_bytes; } +static void ggml_hexagon_precompute_softmax_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_softmax_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); + + float scale = 1.0f; + float max_bias = 0.0f; + memcpy(&scale, &op->op_params[0], sizeof(float)); + memcpy(&max_bias, &op->op_params[1], sizeof(float)); + + kparams->scale = scale; + kparams->max_bias = max_bias; + + const uint32_t n_head = src0->ne[2]; + const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head)); + kparams->n_head = n_head; + kparams->n_head_log2 = n_head_log2; + + if (max_bias > 0.0f && n_head_log2 > 0) { + kparams->m0 = powf(2.0f, -(max_bias) / n_head_log2); + kparams->m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + } else { + kparams->m0 = 1.0f; + kparams->m1 = 1.0f; + } + + kparams->use_src1 = (src1 != nullptr) ? 1 : 0; + kparams->use_f16 = (src1 != nullptr && src1->type == GGML_TYPE_F16) ? 1 : 0; + + const uint32_t ne00 = src0->ne[0]; + const uint32_t ne10 = src1 ? src1->ne[0] : 1; + + struct htp_softmax_vtcm_layout layout; + htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_threads); + + kparams->n_threads = n_threads; + kparams->src0_nrows = src0_nrows; + kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->vtcm_src0_size_per_thread = (uint32_t) layout.src0_bytes_per_thread; + kparams->vtcm_src1_size_per_thread = (uint32_t) layout.src1_bytes_per_thread; + kparams->vtcm_dst_size_per_thread = (uint32_t) layout.dst_bytes_per_thread; + kparams->src0_row_size_aligned = (uint32_t) layout.src0_spad_half_size; + kparams->src1_row_size_aligned = (uint32_t) layout.src1_spad_half_size; + kparams->dst_row_size_aligned = (uint32_t) layout.dst_spad_half_size; + kparams->src0_spad_half_size = (uint32_t) layout.src0_spad_half_size; + kparams->src1_spad_half_size = (uint32_t) layout.src1_spad_half_size; + kparams->dst_spad_half_size = (uint32_t) layout.dst_spad_half_size; + if (!kparams->use_src1) { + kparams->kernel_id = HTP_SOFTMAX_KERNEL_NOMASK; + } else if (kparams->use_f16) { + kparams->kernel_id = HTP_SOFTMAX_KERNEL_MASK_F16; + } else { + kparams->kernel_id = HTP_SOFTMAX_KERNEL_MASK_F32; + } + + if (src0->ne[1] > 0) kparams->div_ne01 = init_fastdiv_values(src0->ne[1]); + if (src0->ne[2] > 0) kparams->div_ne02 = init_fastdiv_values(src0->ne[2]); + const uint32_t ne12 = src1 ? src1->ne[2] : 1; + const uint32_t ne13 = src1 ? src1->ne[3] : 1; + if (ne12 > 0) kparams->div_ne12 = init_fastdiv_values(ne12); + if (ne13 > 0) kparams->div_ne13 = init_fastdiv_values(ne13); +} + +static void ggml_hexagon_precompute_rope_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_rope_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src2 = op->src[2]; + const struct ggml_tensor * dst = op; + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); + const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0; + + struct htp_rope_vtcm_layout layout; + htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors); + + kparams->n_threads = n_threads; + kparams->src0_nrows = src0_nrows; + kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->spad_per_thread = (uint32_t) layout.bytes_per_thread; + kparams->theta_cache_offset = (uint32_t) layout.theta_cache_size_aligned; + kparams->src0_row_size_aligned = (uint32_t) layout.src0_row_size_aligned; + kparams->freq_factors_offset = (uint32_t) (layout.bytes_per_thread * n_threads); + kparams->freq_factors_size = (uint32_t) layout.freq_factors_size_aligned; + + if (src0_nrows > 0) { + kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); + kparams->div_ne1 = init_fastdiv_values(dst->ne[1]); + } +} + +static void ggml_hexagon_precompute_ssm_conv_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_ssm_conv_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const uint32_t d_conv = (uint32_t) src1->ne[0]; + const uint32_t d_inner = (uint32_t) src0->ne[1]; + const uint32_t n_t = (uint32_t) dst->ne[1]; + const uint32_t n_s = (uint32_t) dst->ne[2]; + const uint32_t ncs = (uint32_t) src0->ne[0]; + + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (d_inner + 31) / 32); + + kparams->n_threads = n_threads; + kparams->d_conv = d_conv; + kparams->d_inner = d_inner; + kparams->n_t = n_t; + kparams->n_s = n_s; + + const uint32_t raw_rpt = (d_inner + n_threads - 1) / n_threads; + const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, 32); + kparams->d_inner_per_thread = d_inner_per_thread; + + kparams->src0_row_size_aligned = hex_round_up(ncs * sizeof(float), 128); + kparams->src1_row_size_aligned = hex_round_up(d_conv * sizeof(float), 128); + kparams->dst_row_size_aligned = hex_round_up(d_inner * sizeof(float), 128); + + if (n_t == 1) { + kparams->d_inner_tile = d_inner_per_thread; + + const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128; + const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128); + const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes; + + const uint32_t src0_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128; + const uint32_t src0_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128); + const uint32_t vtcm_src0_per_thread = src0_raw_bytes + src0_T_bytes; + + const uint32_t vtcm_dst_per_thread = hex_round_up(d_inner_per_thread * sizeof(float), 128); + + kparams->vtcm_src0_size_per_thread = vtcm_src0_per_thread; + kparams->vtcm_src1_size_per_thread = vtcm_src1_per_thread; + kparams->vtcm_dst_size_per_thread = vtcm_dst_per_thread; + + kparams->vtcm_src0_size = vtcm_src0_per_thread * n_threads; + kparams->vtcm_src1_size = vtcm_src1_per_thread * n_threads; + kparams->vtcm_dst_size = vtcm_dst_per_thread * n_threads; + kparams->vtcm_size = kparams->vtcm_src0_size + kparams->vtcm_src1_size + kparams->vtcm_dst_size; + } else { + const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128; + const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128); + const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes; + + const size_t vtcm_budget = (sess->vtcm_size > 0 ? sess->vtcm_size / n_threads : (1024 * 1024)); + const size_t avail_for_src0 = vtcm_budget > vtcm_src1_per_thread ? vtcm_budget - vtcm_src1_per_thread : (128 * 1024); + + uint32_t d_inner_tile = (uint32_t)((avail_for_src0 / 2) / (ncs * sizeof(float) + n_t * sizeof(float) + 1)); + d_inner_tile = (d_inner_tile / 32) * 32; + if (d_inner_tile == 0) { + d_inner_tile = 32; + } + if (d_inner_tile > d_inner_per_thread) { + d_inner_tile = d_inner_per_thread; + } + kparams->d_inner_tile = d_inner_tile; + + const uint32_t src0_tile_raw = hex_round_up(d_inner_tile * ncs * sizeof(float), 128) + 128; + const uint32_t src0_tile_T = hex_round_up(ncs * d_inner_tile * sizeof(float), 128); + const uint32_t vtcm_src0_per_thread = src0_tile_raw + src0_tile_T; + + const uint32_t vtcm_dst_per_thread = hex_round_up(d_inner_tile * n_t * sizeof(float), 128); + + kparams->vtcm_src0_size_per_thread = vtcm_src0_per_thread; + kparams->vtcm_src1_size_per_thread = vtcm_src1_per_thread; + kparams->vtcm_dst_size_per_thread = vtcm_dst_per_thread; + + kparams->vtcm_src0_size = vtcm_src0_per_thread * n_threads; + kparams->vtcm_src1_size = vtcm_src1_per_thread * n_threads; + kparams->vtcm_dst_size = vtcm_dst_per_thread * n_threads; + kparams->vtcm_size = kparams->vtcm_src0_size + kparams->vtcm_src1_size + kparams->vtcm_dst_size; + } + + kparams->div_n_threads = init_fastdiv_values(n_threads); +} + +static void ggml_hexagon_precompute_gated_delta_net_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_gdn_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * q = op->src[0]; + const struct ggml_tensor * k = op->src[1]; + const struct ggml_tensor * v = op->src[2]; + const struct ggml_tensor * g = op->src[3]; + const struct ggml_tensor * state = op->src[5]; + + const uint32_t S_v = (uint32_t) v->ne[0]; + const uint32_t H = (uint32_t) v->ne[1]; + const uint32_t n_tokens = (uint32_t) v->ne[2]; + const uint32_t n_seqs = (uint32_t) v->ne[3]; + const uint32_t K = (uint32_t) ggml_get_op_params_i32(op, 0); + + const uint32_t rq3 = (uint32_t) (n_seqs / q->ne[3]); + const uint32_t rk3 = (uint32_t) (n_seqs / k->ne[3]); + const uint32_t total_rows = H * n_seqs; + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, total_rows); + + const bool can_use_hmx = (opt_gdn_select >= 2) && + (sess->n_hmx > 0) && + (S_v % 64 == 0) && + (n_tokens >= HTP_GDN_MIN_TOKENS) && + (g->ne[0] == 1) && + (K == 1); + + struct htp_gdn_hmx_vtcm_layout hmx_layout; + struct htp_gdn_vtcm_layout hvx_layout; + uint32_t n_heads_batch = 1; + + if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, S_v, HTP_GDN_CHUNK_SIZE, total_rows, sess->vtcm_size, n_threads, true, &n_heads_batch)) { + kparams->kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED; + kparams->pipeline = hmx_layout.pipeline ? 1 : 0; + kparams->chunk_size = HTP_GDN_CHUNK_SIZE; + kparams->n_chunks = (n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE; + kparams->n_heads_batch = (uint16_t) n_heads_batch; + kparams->vtcm_size = (uint32_t) hmx_layout.total_bytes; + kparams->state_aligned = (uint32_t) hmx_layout.state_f32_bytes; + kparams->vtcm_per_thread = (uint32_t) (hmx_layout.total_bytes / (n_threads > 0 ? n_threads : 1)); + } else { + htp_gdn_vtcm_layout_build(&hvx_layout, S_v, n_threads); + kparams->kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT; + kparams->pipeline = 0; + kparams->n_heads_batch = 1; + kparams->state_aligned = (uint32_t) hvx_layout.state_aligned; + kparams->vtcm_per_thread = (uint32_t) hvx_layout.bytes_per_thread; + kparams->vtcm_size = (uint32_t) hvx_layout.total_bytes; + } + + kparams->n_threads = n_threads; + kparams->S_v = S_v; + kparams->H = H; + kparams->n_tokens = n_tokens; + kparams->n_seqs = n_seqs; + kparams->K = K; + kparams->total_rows = total_rows; + kparams->rows_per_thread = (total_rows + kparams->n_threads - 1) / kparams->n_threads; + kparams->kda = (g->ne[0] == S_v) ? 1 : 0; + kparams->state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float)); + kparams->state_size_per_snap = S_v * S_v * H * n_seqs; + kparams->scale = 1.0f / sqrtf((float) S_v); + + if (H > 0) kparams->div_H = init_fastdiv_values(H); + if (q->ne[1] > 0) kparams->div_q1 = init_fastdiv_values((uint32_t) q->ne[1]); + if (k->ne[1] > 0) kparams->div_k1 = init_fastdiv_values((uint32_t) k->ne[1]); + if (rq3 > 0) kparams->div_rq3 = init_fastdiv_values(rq3); + if (rk3 > 0) kparams->div_rk3 = init_fastdiv_values(rk3); + if (kparams->n_threads > 0) kparams->div_n_threads = init_fastdiv_values(kparams->n_threads); +} + static void ggml_hexagon_precompute_fused_mmnx_params( const struct ggml_hexagon_session * sess, const struct ggml_tensor * src0, // W0 @@ -4156,6 +5275,7 @@ static void ggml_hexagon_precompute_fused_mmnx_params( struct htp_mm_kernel_params * kparams ) { memset(kparams, 0, sizeof(*kparams)); + kparams->n_threads = sess->n_threads; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; @@ -4176,9 +5296,9 @@ static void ggml_hexagon_precompute_fused_mmnx_params( const size_t vtcm_budget = sess->vtcm_size; const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1); - bool hmx_enabled = (sess->n_hmx > 0) && (opt_mm_select >= 3); + bool hmx_enabled = (sess->n_hmx > 0) && (opt_mm_select >= 2); if (hmx_enabled && ggml_hexagon_matmul_is_hmx_eligible(src0, src1, nullptr, ne01_padded, false, is_batched)) { - if (ggml_hexagon_precompute_hmx_mm_params(sess, src0, src1, nullptr, wtype, ne00_padded, ne01_padded, ne02, ne11, ne12, ne11_padded, false, is_batched, vtcm_budget, kparams)) { + if (ggml_hexagon_precompute_hmx_mm_params(sess, src0, src1, nullptr, wtype, ne00_padded, ne01_padded, ne02, ne11, ne12, ne11_padded, false, is_batched, 0, vtcm_budget, kparams)) { kparams->n_weights = n_weights; goto finalize; } @@ -4191,7 +5311,7 @@ static void ggml_hexagon_precompute_fused_mmnx_params( { const int src1_nrows = ne11 * ne12 * ne13; - const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); const size_t src0_row_size = src0->nb[1]; uint32_t best_n_prefetch = 16; @@ -4213,7 +5333,7 @@ static void ggml_hexagon_precompute_fused_mmnx_params( } struct htp_mm_hvx_vtcm_layout L; - bool try_tiled = (opt_mm_select >= 2); + bool try_tiled = (opt_mm_select >= 1); // Test tiled first htp_mm_hvx_vtcm_layout_build( @@ -4230,19 +5350,8 @@ static void ggml_hexagon_precompute_fused_mmnx_params( kparams->n_prefetch = best_n_prefetch; kparams->n_weights = n_weights; } else { - kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; - size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - - htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT, wtype, ne10, src1_nrows, sess->n_threads, - 0, src0_row_size, flat_src1_row_size, 0, best_n_prefetch, false, true - ); - kparams->vtcm_src0_size = L.src0_bytes; - kparams->vtcm_src1_size = L.src1_bytes; - kparams->vtcm_dst_size = L.dst_bytes; - kparams->vtcm_size = L.total_bytes; - kparams->n_prefetch = best_n_prefetch; - kparams->n_weights = n_weights; + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; } } @@ -4251,7 +5360,7 @@ static void ggml_hexagon_precompute_fused_mmnx_params( kparams->div_ne1 = init_fastdiv_values(ne11); kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1); kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1); - kparams->div_ne11 = init_fastdiv_values(ne11); + kparams->div_ne12 = init_fastdiv_values(ne12); } static void ggml_hexagon_precompute_fused_mmidnx_params( @@ -4262,7 +5371,7 @@ static void ggml_hexagon_precompute_fused_mmidnx_params( int32_t n_weights, struct htp_mm_kernel_params * kparams ) { - ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, 0, kparams); + ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, 0, 0, kparams); kparams->n_weights = n_weights; } @@ -4294,12 +5403,21 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s case GGML_TYPE_Q8_0: case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: - if (src0->ne[0] % 32) { + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q6_K: + if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0)) { return false; } - if (src1->ne[2] != 1 || src1->ne[3] != 1) { - return false; // no broadcasting (for now) + if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q4_K) ? QK_K : 32)) { + return false; + } + + if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) { + return false; + } + if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) { + return false; } if (!src0->buffer) { @@ -4314,6 +5432,9 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) { return false; } + if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) { + return false; + } break; case GGML_TYPE_F32: @@ -4326,6 +5447,9 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) { return false; } + if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) { + return false; + } break; default: @@ -4334,7 +5458,7 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s struct htp_mm_kernel_params kparams; ggml_hexagon_precompute_matmul_params(sess, src0, src1, dst, &kparams); - if ((size_t)kparams.vtcm_size > sess->vtcm_size) { + if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams.vtcm_size > sess->vtcm_size) { HEX_VERBOSE("ggml-hex: %s supported MUL_MAT VTCM size needed (%d) > budget (%zu)\n", sess->c_name(), kparams.vtcm_size, sess->vtcm_size); return false; } @@ -4358,7 +5482,13 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session case GGML_TYPE_Q8_0: case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: - if ((src0->ne[0] % 32)) { + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q6_K: + if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0)) { + return false; + } + + if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q4_K) ? QK_K : 32)) { return false; } @@ -4373,7 +5503,7 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session struct htp_mm_kernel_params kparams; ggml_hexagon_precompute_matmul_params(sess, src0, src1, dst, &kparams); - if ((size_t)kparams.vtcm_size > sess->vtcm_size) { + if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams.vtcm_size > sess->vtcm_size) { HEX_VERBOSE("ggml-hex: %s supported MUL_MAT_ID VTCM size needed (%d) > budget (%zu)\n", sess->c_name(), kparams.vtcm_size, sess->vtcm_size); return false; } @@ -4416,37 +5546,42 @@ static bool ggml_hexagon_supported_binary(const struct ggml_hexagon_session * se return false; } - return true; - - GGML_UNUSED(sess); + struct htp_binary_kernel_params kparams; + return ggml_hexagon_precompute_binary_params(sess, op_remap_to_htp(op), src0, src1, dst, &kparams); } static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { const struct ggml_tensor * src0 = op->src[0]; const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * src2 = op->src[2]; const struct ggml_tensor * dst = op; - if (src0->type != GGML_TYPE_F32) { + if (!src2) { return false; } - if (src1->type != GGML_TYPE_F32) { + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) { return false; } - if (dst->type != GGML_TYPE_F32) { + if (!ggml_are_same_shape(src0, dst)) { return false; } - if (!ggml_are_same_shape(src0, dst)) { + if (src1->ne[0] != src0->ne[0] || src1->ne[2] != 1 || src1->ne[3] != 1) { + return false; + } + if (src2->ne[0] != src0->ne[1] || src2->ne[1] != src0->ne[2]) { + return false; + } + if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || src2->nb[0] != sizeof(int32_t)) { return false; } - // REVISIT: add support for non-contigiuos tensors + // REVISIT: add support for non-contiguous tensors if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { return false; } - return true; - - GGML_UNUSED(sess); + struct htp_binary_kernel_params kparams; + return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams); } static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -4574,6 +5709,10 @@ static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * s return false; } + if (src0->ne[2] > 512) { + return false; + } + if (src1) { if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) { return false; @@ -4619,6 +5758,14 @@ static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * s return false; } + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); + struct htp_softmax_vtcm_layout layout; + htp_softmax_vtcm_layout_build(&layout, src0->ne[0], src1 ? src1->ne[0] : 1, src1 != nullptr, src1 && src1->type == GGML_TYPE_F16, n_threads); + if (layout.total_bytes > sess->vtcm_size) { + return false; + } + return true; GGML_UNUSED(sess); @@ -4662,11 +5809,12 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * } } - if (src0->type != GGML_TYPE_F32 && src0->ne[0] < 32) { + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->ne[0] < 32) { return false; } - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_Q8_0) { + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && + src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) { return false; } @@ -4674,7 +5822,12 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * return false; } - if (dst->type != GGML_TYPE_F32) { + if (src0->type == GGML_TYPE_I32) { + if (dst->type != GGML_TYPE_I32) { + return false; + } + } + else if (dst->type != GGML_TYPE_F32) { return false; } @@ -4705,57 +5858,109 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s GGML_UNUSED(sess); } -static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const int32_t * op_params = &op->op_params[0]; +static bool ggml_hexagon_supported_top_k(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { + const struct ggml_tensor * src0 = op->src[0]; // values + const struct ggml_tensor * dst = op; // indices - // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems) - if (op_params[15] % 32 != 0) { + if (src0->type != GGML_TYPE_F32) { return false; } - int mode = op_params[2]; - - // n_dims == ne0/2, so the rotation spans the full row - if (mode == GGML_ROPE_TYPE_VISION) { - const int n_dims = op_params[1]; - if (n_dims != (int) (op->src[0]->ne[0] / 2)) { - return false; - } + if (dst->type != GGML_TYPE_I32) { + return false; } - if (mode & 1) { + + // Single row uses the threaded chunk+merge path. Multi-row uses one full + // buffer per thread, so it keeps the tighter 64K cap. + const bool single_row = (src0->ne[1] == 1 && src0->ne[2] == 1 && src0->ne[3] == 1); + const int64_t max_ne00 = single_row ? (256*1024) : (64*1024); + + if (src0->ne[0] > max_ne00) { return false; } + return true; +} + +static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { const struct ggml_tensor * src0 = op->src[0]; const struct ggml_tensor * src1 = op->src[1]; const struct ggml_tensor * src2 = op->src[2]; const struct ggml_tensor * dst = op; - if (src0->type != GGML_TYPE_F32) { - return false; // FIXME: add support for GGML_TYPE_F16 for src0 + if (!ggml_are_same_shape(src0, dst)) { + return false; } - if (dst->type != GGML_TYPE_F32) { + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) { return false; } - if (src1->type != GGML_TYPE_I32) { + + if (src0->ne[0] <= 0) { return false; } - if (src2) { - if (src2->type != GGML_TYPE_F32) { + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + if (src0_nrows == 0) { + return false; + } + + const int32_t * op_params = &op->op_params[0]; + const int n_dims = op_params[1]; + const int mode = op_params[2]; + const int n_offs = op_params[15]; + + // llama probes weight placement with a dummy rope where every param is 0 (llama-model-loader.cpp). + // Rejecting it puts rope_freqs on the CPU, which then splits the graph at every full-attention layer. + if (n_dims < 0 || n_dims % 2 != 0) { + return false; + } + + // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems) + if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) { + return false; + } + + float freq_base; + memcpy(&freq_base, op_params + 5, sizeof(float)); + if (freq_base < 0.0f) { + return false; + } + + if (mode != GGML_ROPE_TYPE_NORMAL && + mode != GGML_ROPE_TYPE_NEOX && + mode != GGML_ROPE_TYPE_MROPE && + mode != GGML_ROPE_TYPE_VISION && + mode != GGML_ROPE_TYPE_IMROPE) { + return false; + } + + const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0; + + // n_dims == ne0/2, so the rotation spans the full row + if (mode == GGML_ROPE_TYPE_VISION) { + if (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0) { return false; } - int n_dims = op_params[1]; - if (src2->ne[0] < (n_dims / 2)) { + } + + if (is_mrope) { + const int32_t * sections = op_params + 11; + if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) { return false; } } + const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2]; + if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) { + return false; + } + if (src2) { - if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(src2)) { + if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2)) { return false; } - } else { - if (!ggml_is_contiguous(src1)) { + if (src2->ne[0] < (n_dims / 2)) { return false; } } @@ -4768,9 +5973,17 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) { return false; } - return true; - GGML_UNUSED(sess); + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); + const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0; + + struct htp_rope_vtcm_layout layout; + htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors); + if (layout.total_bytes > sess->vtcm_size) { + return false; + } + + return true; } static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -4788,11 +6001,14 @@ static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * return false; // src0 should be effectively 3D } - const int d_conv = src1->ne[0]; + const int d_conv = src1->ne[0]; const int d_inner = src0->ne[1]; const int n_t = dst->ne[1]; const int n_s = dst->ne[2]; + if (d_conv == 0 || d_conv > 32 || d_inner == 0) { + return false; + } if (src0->ne[0] != d_conv - 1 + n_t || src0->ne[1] != d_inner || src0->ne[2] != n_s) { return false; } @@ -4809,20 +6025,19 @@ static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * return false; } - return true; + struct htp_ssm_conv_kernel_params kparams; + ggml_hexagon_precompute_ssm_conv_params(sess, src0, src1, dst, &kparams); + if ((size_t) kparams.vtcm_size > sess->vtcm_size) { + return false; + } - GGML_UNUSED(sess); + return true; } static bool ggml_hexagon_supported_im2col(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { const struct ggml_tensor * src1 = op->src[1]; const struct ggml_tensor * dst = op; - const bool is_2D = ((const int32_t *) op->op_params)[6] == 1; - if (!is_2D) { - return false; - } - // For now support F32->F32 and F32->F16 only. if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) { return false; @@ -4832,13 +6047,6 @@ static bool ggml_hexagon_supported_im2col(const struct ggml_hexagon_session * se return false; } - // For now keep padded OPs on CPU. Will revisit once we expand coverage past patch-embed OPs. - const int32_t p0 = ((const int32_t *) op->op_params)[2]; - const int32_t p1 = ((const int32_t *) op->op_params)[3]; - if (p0 != 0 || p1 != 0) { - return false; - } - GGML_UNUSED(sess); return true; } @@ -4851,6 +6059,14 @@ static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, return false; } + const int32_t lp0 = ((const int32_t *) op->op_params)[0]; + const int32_t rp0 = ((const int32_t *) op->op_params)[1]; + const int32_t circular = ((const int32_t *) op->op_params)[8]; + + if (circular && (lp0 > src0->ne[0] || rp0 > src0->ne[0])) { + return false; + } + return true; GGML_UNUSED(sess); @@ -4902,10 +6118,6 @@ static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session * const struct ggml_tensor * src1 = op->src[1]; // B const struct ggml_tensor * dst = op; // X - if (!src0 || !src1) { - return false; - } - if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { return false; } @@ -4973,12 +6185,14 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) { case GGML_OP_SET_ROWS: return HTP_OP_SET_ROWS; case GGML_OP_SUM_ROWS: return HTP_OP_SUM_ROWS; case GGML_OP_ARGSORT: return HTP_OP_ARGSORT; + case GGML_OP_TOP_K: return HTP_OP_TOP_K; case GGML_OP_NORM: return HTP_OP_NORM; case GGML_OP_L2_NORM: return HTP_OP_L2_NORM; case GGML_OP_RMS_NORM: return HTP_OP_RMS_NORM; case GGML_OP_CONCAT: return HTP_OP_CONCAT; case GGML_OP_SCALE: return HTP_OP_SCALE; case GGML_OP_CLAMP: return HTP_OP_CLAMP; + case GGML_OP_LEAKY_RELU: return HTP_OP_LEAKY_RELU; case GGML_OP_SQR: return HTP_OP_SQR; case GGML_OP_SQRT: return HTP_OP_SQRT; case GGML_OP_LOG: return HTP_OP_UNARY_LOG; @@ -4994,6 +6208,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) { case GGML_OP_TRI: return HTP_OP_TRI; case GGML_OP_PAD: return HTP_OP_PAD; case GGML_OP_IM2COL: return HTP_OP_IM2COL; + case GGML_OP_ROLL: return HTP_OP_ROLL; case GGML_OP_UNARY: switch (ggml_get_unary_op(t)) { @@ -5006,6 +6221,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) { case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS; case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH; case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS; + case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU; default: break; } @@ -5017,6 +6233,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) { case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI; case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP; case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU; + case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK; default: break; } break; @@ -5053,11 +6270,11 @@ static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struc return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D; } - if (!ggml_hexagon_is_repack_type(src0->type)) { - return false; + if (!ggml_hexagon_is_repack_type(src0->type) || src0->type == GGML_TYPE_Q6_K) { + return false; // Q6_K has no fused HVX kernel } - return kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; + return kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW; } static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) { @@ -5073,7 +6290,7 @@ static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const st } static bool is_mergeable_mul_mat(const ggml_tensor * t) { - if (!t || t->op != GGML_OP_MUL_MAT) return false; + if (t->op != GGML_OP_MUL_MAT) return false; const ggml_tensor * src0 = t->src[0]; const ggml_tensor * src1 = t->src[1]; @@ -5084,7 +6301,7 @@ static bool is_mergeable_mul_mat(const ggml_tensor * t) { return ggml_hexagon_is_hmx_weight_type(src0->type); } - return ggml_hexagon_is_repack_type(src0->type); + return ggml_hexagon_is_repack_type(src0->type) && src0->type != GGML_TYPE_Q6_K; } static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2) { @@ -5107,7 +6324,7 @@ static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor } static bool is_mergeable_mul_mat_id(const ggml_tensor * t) { - if (!t || t->op != GGML_OP_MUL_MAT_ID) return false; + if (t->op != GGML_OP_MUL_MAT_ID) return false; const ggml_tensor * src0 = t->src[0]; return ggml_hexagon_is_repack_type(src0->type); @@ -5141,6 +6358,10 @@ static bool is_mergeable_mul_mat_id_pair(const ggml_tensor * n1, const ggml_tens static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { auto sess = static_cast<ggml_hexagon_session *>(backend->context); + if (sess->last_error > HTP_STATUS_OK) { + return GGML_STATUS_FAILED; + } + HEX_VERBOSE("ggml-hex: %s graph-compute n_nodes %d\n", sess->c_name(), graph->n_nodes); const std::vector<htp_opnode> * nodes_ptr = nullptr; @@ -5156,6 +6377,8 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg auto * extra = (ggml_hexagon_tensor_extra *) graph->nodes[i]->extra; if (!extra) continue; + extra->flags &= ~GGML_HEXAGON_TENSOR_FUSEABLE; + if (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE; } else if (graph->nodes[i]->op == GGML_OP_MUL_MAT || graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) { @@ -5181,6 +6404,11 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg node.node->src[0], node.node->src[1], node.node, (struct htp_mm_kernel_params *)node.kernel_params ); + } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { + const ggml_tensor * src1 = node.node->src[1]; + GGML_ASSERT(ggml_hexagon_precompute_binary_params(sess, + node.opcode, node.node->src[0], src1, node.node, + (struct htp_binary_kernel_params *) node.kernel_params)); } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { ggml_hexagon_precompute_flash_attn_params(sess, node.node, @@ -5204,6 +6432,26 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg node.node->src[0], node.node->src[1], node.dst(), (struct htp_set_rows_kernel_params *)node.kernel_params ); + } else if (node.opcode == HTP_OP_ROPE) { + ggml_hexagon_precompute_rope_params(sess, + node.node, + (struct htp_rope_kernel_params *)node.kernel_params + ); + } else if (node.opcode == HTP_OP_SSM_CONV) { + ggml_hexagon_precompute_ssm_conv_params(sess, + node.node->src[0], node.node->src[1], node.dst(), + (struct htp_ssm_conv_kernel_params *)node.kernel_params + ); + } else if (node.opcode == HTP_OP_SOFTMAX) { + ggml_hexagon_precompute_softmax_params(sess, + node.node, + (struct htp_softmax_kernel_params *)node.kernel_params + ); + } else if (node.opcode == HTP_OP_GATED_DELTA_NET) { + ggml_hexagon_precompute_gated_delta_net_params(sess, + node.node, + (struct htp_gdn_kernel_params *)node.kernel_params + ); } computed_nodes.push_back(std::move(node)); } @@ -5222,6 +6470,10 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg sess->enqueue_op(node); } + if (sess->last_error > HTP_STATUS_OK) { + return GGML_STATUS_FAILED; + } + return GGML_STATUS_SUCCESS; } @@ -5231,7 +6483,10 @@ static void ggml_backend_hexagon_synchronize(ggml_backend_t backend) { HEX_VERBOSE("ggml-hex: %s synchronize\n", sess->c_name()); // Wait until all pending ops complete - sess->flush(); + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s synchronize failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); + } } enum ggml_hexagon_mem_range_type { @@ -5466,27 +6721,40 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr GGML_UNUSED(backend); } +static uint64_t ggml_hexagon_session_key(const ggml_hexagon_session * sess) { + return ((uint64_t) (uint32_t) sess->phys_idx << 32) | (uint32_t) sess->virt_idx; +} + static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context); auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context); - auto sbuf_dst = (ggml_hexagon_shared_buffer *) dst->buffer->context; - if (sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; - uint32_t fence_seq = sess_dst->fence_seq++; - if (sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; + sess_src->mmap_tensor(src); + auto sbuf_dst = sess_dst->mmap_tensor(dst); + + if (!sess_src->clone_buffer(sbuf_dst)) { return false; } + + const uint64_t src_key = ggml_hexagon_session_key(sess_src); + auto & fence_slot = sess_dst->cpy_fence_slots[src_key]; + if (!fence_slot) { + fence_slot = (volatile uint32_t *) sess_dst->alloc_fence(1); + } + + if (!sess_src->clone_buffer(sess_dst->fence_buf)) { return false; } - volatile uint32_t * fence = (volatile uint32_t *) sbuf_dst->alloc_fence(); + if (++sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; + uint32_t fence_seq = sess_dst->fence_seq; - HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq %u\n", + HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq 0x%x\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src), fence_seq); - // dummy extra (must be static) + // dummy fence extra (must be static) static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; ggml_tensor fence_tensor {}; - fence_tensor.buffer = dst->buffer; + fence_tensor.buffer = &sess_dst->fence_buf->backend_buffer; fence_tensor.extra = &fence_extra; - fence_tensor.data = (void *) fence; + fence_tensor.data = (void *) fence_slot; fence_tensor.type = GGML_TYPE_I32; fence_tensor.ne[0] = 1; fence_tensor.ne[1] = 1; @@ -5499,9 +6767,9 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_ fence_tensor.op = GGML_OP_NONE; sess_src->enqueue_cpy(src, dst, &fence_tensor, fence_seq); - sess_dst->enqueue_fence(&fence_tensor, fence_seq); + sess_dst->enqueue_fence(&fence_tensor, fence_seq, /* wait = */ true); - sess_dst->add_sync_peer(sess_src); + sess_dst->add_peer(sess_src); return true; } @@ -5509,15 +6777,17 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_ static bool ggml_hexagon_cpy_tensor_async_virt(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context); auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context); - auto sbuf_dst = (ggml_hexagon_shared_buffer *) dst->buffer->context; - if (!sess_src->clone_buffer(sbuf_dst)) { return false; } + auto sbuf_src = sess_src->mmap_tensor(src); + sess_dst->mmap_tensor(dst); + + if (!sess_dst->clone_buffer(sbuf_src)) { return false; } HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src)); - sess_src->enqueue_cpy(src, dst); - sess_src->flush(true); + sess_dst->enqueue_cpy(src, dst); + sess_dst->add_peer(sess_src); return true; } @@ -5527,7 +6797,14 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg return false; } - *(ggml_hexagon_tensor_extra *) dst->extra = *(const ggml_hexagon_tensor_extra *) src->extra; + // FIXME: ggml-meta needs to call init_tensor on auxiliary tensors + if (!dst->extra) { + ggml_backend_buffer_init_tensor(dst->buffer, dst); + } + + auto * dst_extra = static_cast<ggml_hexagon_tensor_extra *>(dst->extra); + const auto * src_extra = static_cast<const ggml_hexagon_tensor_extra *>(src->extra); + dst_extra->flags = src_extra->flags & ~GGML_HEXAGON_TENSOR_FUSEABLE; auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context); auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context); @@ -5535,7 +6812,6 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg if (sess_src == sess_dst) { HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src)); sess_src->enqueue_cpy(src, dst); - sess_src->flush_batch(); return true; } @@ -5546,8 +6822,30 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg } static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_dev_t dev) { + auto dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context); + auto sess = dev_ctx->session(); + ggml_hexagon_event * hex_event = new ggml_hexagon_event(); - HEX_VERBOSE("ggml-hex: %s event-new : event %p\n", ggml_backend_dev_name(dev), (void *)hex_event); + hex_event->fence_sess = sess; + hex_event->sess = sess; + hex_event->fence_slot = (volatile uint32_t *) sess->alloc_fence(1); + + static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; + hex_event->fence_tensor.buffer = &sess->fence_buf->backend_buffer; + hex_event->fence_tensor.extra = &fence_extra; + hex_event->fence_tensor.data = (void *) hex_event->fence_slot; + hex_event->fence_tensor.type = GGML_TYPE_I32; + hex_event->fence_tensor.ne[0] = 1; + hex_event->fence_tensor.ne[1] = 1; + hex_event->fence_tensor.ne[2] = 1; + hex_event->fence_tensor.ne[3] = 1; + hex_event->fence_tensor.nb[0] = sizeof(int32_t); + hex_event->fence_tensor.nb[1] = sizeof(int32_t); + hex_event->fence_tensor.nb[2] = sizeof(int32_t); + hex_event->fence_tensor.nb[3] = sizeof(int32_t); + hex_event->fence_tensor.op = GGML_OP_NONE; + + HEX_VERBOSE("ggml-hex: %s event-new : event %p fence %p\n", ggml_backend_dev_name(dev), (void *)hex_event, (void *)hex_event->fence_slot); return new ggml_backend_event { /* .device = */ dev, @@ -5555,49 +6853,83 @@ static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_d }; } -static void ggml_backend_hexagon_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) { - GGML_UNUSED(dev); - - if (event == nullptr) { +static void ggml_hexagon_event_synchronize(ggml_backend_dev_t dev, ggml_hexagon_event * hex_event) { + if (hex_event->seq == 0) { return; } - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; + HEX_VERBOSE("ggml-hex: %s event-synchronize : event %p seq 0x%x fence %p\n", + ggml_backend_dev_name(dev), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); + + auto * fence = reinterpret_cast<const volatile std::atomic<uint32_t> *>(hex_event->fence_slot); + + if ((int32_t)(fence[0].load(std::memory_order_relaxed) - hex_event->seq) < 0) { + hex_event->sess->flush_async(); + } + + while (true) { + if ((int32_t)(fence[0].load(std::memory_order_acquire) - hex_event->seq) >= 0) { + uint32_t status = fence[1].load(std::memory_order_acquire); + if (status > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s event-synchronize failed : dsp-error %s\n", + hex_event->sess->c_name(), status_to_str(status)); + } + break; + } + std::this_thread::yield(); + } +} + +static void ggml_backend_hexagon_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) { + auto * hex_event = static_cast<ggml_hexagon_event *>(event->context); + ggml_hexagon_event_synchronize(dev, hex_event); HEX_VERBOSE("ggml-hex: %s event-free : event %p\n", ggml_backend_dev_name(dev), (void *)hex_event); + hex_event->fence_sess->free_fence((void *) hex_event->fence_slot, 1); delete hex_event; delete event; } static void ggml_backend_hexagon_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) { - GGML_UNUSED(dev); - - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; - HEX_VERBOSE("ggml-hex: %s event-synchronize : event %p seq %llu\n", - ggml_backend_dev_name(dev), (void *)hex_event, (unsigned long long)hex_event->seq); - if (hex_event->sess != nullptr) { - hex_event->sess->wait_event(hex_event->seq); - } + auto * hex_event = static_cast<ggml_hexagon_event *>(event->context); + ggml_hexagon_event_synchronize(dev, hex_event); } static void ggml_backend_hexagon_event_record(ggml_backend_t backend, ggml_backend_event_t event) { auto sess = static_cast<ggml_hexagon_session *>(backend->context); - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; + auto hex_event = static_cast<ggml_hexagon_event *>(event->context); + if (++sess->fence_seq == 0) sess->fence_seq = 1; hex_event->sess = sess; - hex_event->seq = sess->record_event(); - HEX_VERBOSE("ggml-hex: %s event-record : event %p seq %llu\n", - sess->c_name(), (void *)hex_event, (unsigned long long)hex_event->seq); + hex_event->seq = sess->fence_seq; + + sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ false); + + HEX_VERBOSE("ggml-hex: %s event-record : event %p seq 0x%x fence %p\n", + sess->c_name(), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); } static void ggml_backend_hexagon_event_wait(ggml_backend_t backend, ggml_backend_event_t event) { - GGML_UNUSED(backend); + auto sess = static_cast<ggml_hexagon_session *>(backend->context); + auto hex_event = static_cast<ggml_hexagon_event *>(event->context); + + if (hex_event->seq == 0) { + return; + } - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; - if (hex_event->sess != nullptr) { - HEX_VERBOSE("ggml-hex: %s event-wait : event %p seq %llu\n", - hex_event->sess->c_name(), (void *)hex_event, (unsigned long long)hex_event->seq); - hex_event->sess->wait_event(hex_event->seq); + HEX_VERBOSE("ggml-hex: %s event-wait : event %p seq 0x%x fence %p\n", + sess->c_name(), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); + + // same physical NPU runs sequentially in FIFO order + if (sess->phys_idx == hex_event->sess->phys_idx) { + if (sess != hex_event->sess) { + sess->add_peer(hex_event->sess); + } + return; } + + sess->clone_buffer(hex_event->fence_sess->fence_buf); + sess->add_peer(hex_event->sess); + sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ true); } static void ggml_backend_hexagon_set_tensor_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { @@ -5611,7 +6943,10 @@ static void ggml_backend_hexagon_get_tensor_async(ggml_backend_t backend, const auto sess = static_cast<ggml_hexagon_session *>(backend->context); HEX_VERBOSE("ggml-hex: %s get-tensor-async %s : data %p offset %zu size %zu usage %d\n", sess->c_name(), tensor->name, data, offset, size, tensor->buffer ? (int) tensor->buffer->usage : -1); - sess->flush(true); + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s get-tensor-async failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); + } ggml_backend_tensor_get(tensor, data, offset, size); } @@ -5640,7 +6975,10 @@ static void ggml_backend_hexagon_get_tensor_2d_async(ggml_backend_t backend, auto sess = static_cast<ggml_hexagon_session *>(backend->context); HEX_VERBOSE("ggml-hex: %s get-tensor-2d-async %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, tensor->buffer ? (int) tensor->buffer->usage : -1); - sess->flush(true); + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s get-tensor-2d-async failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); + } ggml_backend_tensor_get_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data); } @@ -5828,6 +7166,31 @@ static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess GGML_UNUSED(sess); } +static bool ggml_hexagon_supported_roll(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { + GGML_UNUSED(sess); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + if (!ggml_are_same_shape(src0, dst)) { + return false; + } + + if (src0->nb[0] != ggml_type_size(src0->type) || dst->nb[0] != ggml_type_size(dst->type)) { + return false; + } + + if (!ggml_is_contiguous(dst)) { + return false; + } + + return true; +} + static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { auto dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context); auto sess = dev_ctx->session(); @@ -5871,6 +7234,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_RMS_NORM: case GGML_OP_SCALE: case GGML_OP_CLAMP: + case GGML_OP_LEAKY_RELU: supp = ggml_hexagon_supported_unary(sess, op); break; @@ -5899,6 +7263,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_GELU: case GGML_UNARY_OP_GELU_QUICK: + case GGML_UNARY_OP_RELU: supp = ggml_hexagon_supported_unary(sess, op); break; default: @@ -5913,6 +7278,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons case GGML_GLU_OP_SWIGLU_OAI: case GGML_GLU_OP_SWIGLU_CLAMP: case GGML_GLU_OP_GEGLU: + case GGML_GLU_OP_GEGLU_QUICK: supp = ggml_hexagon_supported_activations(sess, op); break; default: @@ -5953,6 +7319,10 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons supp = ggml_hexagon_supported_argsort(sess, op); break; + case GGML_OP_TOP_K: + supp = ggml_hexagon_supported_top_k(sess, op); + break; + case GGML_OP_SSM_CONV: supp = ggml_hexagon_supported_ssm_conv(sess, op); break; @@ -5993,6 +7363,10 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons supp = ggml_hexagon_supported_pad(sess, op); break; + case GGML_OP_ROLL: + supp = ggml_hexagon_supported_roll(sess, op); + break; + default: break; } @@ -6004,13 +7378,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons static bool ggml_backend_hexagon_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { auto dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context); - // Technically we can clone hexagon buffers from any session but for some reason the output is garbled with layer-split, - // tensor-split works correctly, so it needs mode debugging and investigation. For now accept only our own buffers. -#if 0 - bool supp = (buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment); -#else bool supp = (buft == &dev_ctx->host_buffer_type) || (buft == &dev_ctx->buffer_type); -#endif HEX_VERBOSE("ggml-hex: %s device-supports-buft %s %s\n", dev_ctx->c_name(), ggml_backend_buft_name(buft), supp ? "yes" : "no"); return supp; @@ -6039,10 +7407,23 @@ static const struct ggml_backend_device_i ggml_backend_hexagon_device_i = { ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) { GGML_LOG_INFO("ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev %zu\n", opt_ndev); - GGML_LOG_INFO("ggml-hex: Hexagon Arch version v%d\n", opt_arch); + GGML_LOG_INFO("ggml-hex: Hexagon Arch version v%d, DMA64 %s\n", opt_arch, opt_dma64 ? "enabled" : "disabled"); // Create devices for (size_t i = 0; i < opt_ndev; i++) { + const auto & cfg = opt_device_configs[i]; + if (cfg.mdev_group.empty()) { + GGML_LOG_INFO("ggml-hex: device %zu: %s (phys=%d, virt=%d, domain=%s:%d)\n", + i, cfg.name.c_str(), cfg.physical_idx, cfg.virtual_idx, cfg.domain_name.c_str(), cfg.domain_id); + } else { + std::string peers_str; + for (const auto & p : cfg.mdev_group) { + if (!peers_str.empty()) peers_str += ", "; + peers_str += p.name + " (phys=" + std::to_string(p.physical_idx) + ")"; + } + GGML_LOG_INFO("ggml-hex: device %zu: %s (phys=%d, virt=%d, domain=%s:%d) [mdev peers: %s]\n", + i, cfg.name.c_str(), cfg.physical_idx, cfg.virtual_idx, cfg.domain_name.c_str(), cfg.domain_id, peers_str.c_str()); + } devices[i].iface = ggml_backend_hexagon_device_i; devices[i].reg = reg; devices[i].context = new ggml_backend_hexagon_device_context(i, opt_device_configs[i], &devices[i]); @@ -6093,17 +7474,51 @@ static void * ggml_backend_hexagon_comm_init(ggml_backend_t * backends, size_t n } } + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast<ggml_hexagon_session *>(backends[i]->context); + for (size_t j = i + 1; j < n_backends; j++) { + auto sess_j = static_cast<ggml_hexagon_session *>(backends[j]->context); + if (sess_i->phys_idx == sess_j->phys_idx) { + return nullptr; + } + } + } + auto * ctx = new ggml_backend_hexagon_comm_context(); ctx->backends.assign(backends, backends + n_backends); ctx->n_backends = n_backends; - ctx->fence_seq = (((uintptr_t) ctx) & 0xFFFF) | 1; + + static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast<ggml_hexagon_session *>(backends[i]->context); + ctx->fence_slots[i] = (volatile uint32_t *) sess_i->alloc_fence(1); + ctx->fence_tensors[i] = {}; + ctx->fence_tensors[i].buffer = &sess_i->fence_buf->backend_buffer; + ctx->fence_tensors[i].extra = &fence_extra; + ctx->fence_tensors[i].data = (void *) ctx->fence_slots[i]; + ctx->fence_tensors[i].type = GGML_TYPE_I32; + ctx->fence_tensors[i].ne[0] = 4; + ctx->fence_tensors[i].ne[1] = 1; + ctx->fence_tensors[i].ne[2] = 1; + ctx->fence_tensors[i].ne[3] = 1; + ctx->fence_tensors[i].nb[0] = sizeof(int32_t); + ctx->fence_tensors[i].nb[1] = sizeof(int32_t); + ctx->fence_tensors[i].nb[2] = sizeof(int32_t); + ctx->fence_tensors[i].nb[3] = sizeof(int32_t); + ctx->fence_tensors[i].op = GGML_OP_NONE; + } return ctx; } static void ggml_backend_hexagon_comm_free(void * comm_ctx_v) { if (!comm_ctx_v) return; - delete static_cast<ggml_backend_hexagon_comm_context *>(comm_ctx_v); + auto * ctx = static_cast<ggml_backend_hexagon_comm_context *>(comm_ctx_v); + for (size_t i = 0; i < ctx->n_backends; i++) { + auto sess_i = static_cast<ggml_hexagon_session *>(ctx->backends[i]->context); + sess_i->free_fence((void *) ctx->fence_slots[i], 1); + } + delete ctx; } static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) { @@ -6113,6 +7528,16 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct if (n_backends < 2 || n_backends > 4) return false; + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast<ggml_hexagon_session *>(comm_ctx->backends[i]->context); + for (size_t j = i + 1; j < n_backends; j++) { + auto sess_j = static_cast<ggml_hexagon_session *>(comm_ctx->backends[j]->context); + if (sess_i->phys_idx == sess_j->phys_idx) { + return false; + } + } + } + for (size_t i = 0; i < n_backends; i++) { if (!tensors[i] || !tensors[i]->buffer || !ggml_backend_buffer_is_hexagon(tensors[i]->buffer)) { return false; @@ -6140,42 +7565,28 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct } } - if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; - uint32_t fence_seq_entry = comm_ctx->fence_seq++; - if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; - uint32_t fence_seq_exit = comm_ctx->fence_seq++; - if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; - - volatile uint32_t * fences[GGML_HEXAGON_MAX_SESSIONS]; - for (size_t i = 0; i < n_backends; i++) { - auto sbuf = (ggml_hexagon_shared_buffer *) tensors[i]->buffer->context; - fences[i] = (volatile uint32_t *) sbuf->alloc_fence(); + uint32_t max_seq = static_cast<ggml_hexagon_session *>(comm_ctx->backends[0]->context)->fence_seq; + for (size_t i = 1; i < n_backends; i++) { + auto sess_i = static_cast<ggml_hexagon_session *>(comm_ctx->backends[i]->context); + if ((int32_t)(sess_i->fence_seq - max_seq) > 0) { + max_seq = sess_i->fence_seq; + } } + if (++max_seq == 0) max_seq = 1; + uint32_t fence_seq_entry = max_seq; + if (++max_seq == 0) max_seq = 1; + uint32_t fence_seq_exit = max_seq; - static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; - ggml_tensor fence_tensors[GGML_HEXAGON_MAX_SESSIONS]; for (size_t i = 0; i < n_backends; i++) { - fence_tensors[i] = {}; - fence_tensors[i].buffer = tensors[i]->buffer; - fence_tensors[i].extra = &fence_extra; - fence_tensors[i].data = (void *) fences[i]; - fence_tensors[i].type = GGML_TYPE_I32; - fence_tensors[i].ne[0] = 4; - fence_tensors[i].ne[1] = 1; - fence_tensors[i].ne[2] = 1; - fence_tensors[i].ne[3] = 1; - fence_tensors[i].nb[0] = sizeof(int32_t); - fence_tensors[i].nb[1] = sizeof(int32_t); - fence_tensors[i].nb[2] = sizeof(int32_t); - fence_tensors[i].nb[3] = sizeof(int32_t); - fence_tensors[i].op = GGML_OP_NONE; + auto sess_i = static_cast<ggml_hexagon_session *>(comm_ctx->backends[i]->context); + sess_i->fence_seq = max_seq; } std::vector<const ggml_tensor *> data_tensors(n_backends); std::vector<const ggml_tensor *> sync_tensors(n_backends); for (size_t i = 0; i < n_backends; i++) { data_tensors[i] = tensors[i]; - sync_tensors[i] = &fence_tensors[i]; + sync_tensors[i] = &comm_ctx->fence_tensors[i]; } for (size_t r = 0; r < n_backends; r++) { @@ -6183,7 +7594,7 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct sess->enqueue_allreduce(tensors[r], data_tensors, sync_tensors, (uint32_t) r, (uint32_t) n_backends, fence_seq_entry, fence_seq_exit); for (size_t j = 0; j < n_backends; j++) { if (r != j) { - sess->add_sync_peer(static_cast<ggml_hexagon_session *>(comm_ctx->backends[j]->context)); + sess->add_peer(static_cast<ggml_hexagon_session *>(comm_ctx->backends[j]->context)); } } } @@ -6191,8 +7602,23 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct return true; } +static ggml_backend_buffer_type_t ggml_backend_hexagon_split_buffer_type(int main_device, const float * tensor_split) { + GGML_UNUSED(tensor_split); + auto reg = ggml_backend_hexagon_reg(); + auto dev = ggml_backend_reg_dev_get(reg, main_device); + if (!dev) { + dev = ggml_backend_reg_dev_get(reg, 0); + } + if (!dev) return nullptr; + auto dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context); + return &dev_ctx->buffer_type; +} + static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, const char * name) { GGML_UNUSED(reg); + if (strcmp(name, "ggml_backend_split_buffer_type") == 0) { + return (void *) ggml_backend_hexagon_split_buffer_type; + } if (strcmp(name, "ggml_backend_comm_init") == 0) { return (void *) ggml_backend_hexagon_comm_init; } @@ -6225,6 +7651,41 @@ template<typename T, int BASE=10> std::string vec_to_str(std::vector<T> v) { return str; } +static void ggml_hexagon_resolve_device_domain(ggml_hexagon_device_config & cfg, bool discovery_supported, const std::unordered_map<int, fastrpc_domain> & cdsp_map) { + if (discovery_supported) { + auto it = cdsp_map.find(cfg.physical_idx); + if (it != cdsp_map.end()) { + cfg.domain_id = it->second.id; + cfg.domain_name = it->second.name; + } else { + GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not found on device (%zu CDSP core(s) available)\n", + cfg.physical_idx, cdsp_map.size()); + cfg.domain_id = -1; + cfg.domain_name = ""; + } + } else { + switch (cfg.physical_idx) { + case 0: + cfg.domain_id = 3; + cfg.domain_name = CDSP_DOMAIN_NAME; + break; + case 1: + cfg.domain_id = 4; + cfg.domain_name = "cdsp1"; + break; + default: + GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not supported without dynamic discovery\n", + cfg.physical_idx); + cfg.domain_id = -1; + cfg.domain_name = ""; + break; + } + } + for (auto & sub_cfg : cfg.mdev_group) { + ggml_hexagon_resolve_device_domain(sub_cfg, discovery_supported, cdsp_map); + } +} + // Enumerate NPU (aka CDSP) domains via FASTRPC_GET_DOMAINS if supported, // and populate domain_id and domain_name for all configured devices. static void ggml_hexagon_discover_devices() { @@ -6271,36 +7732,7 @@ static void ggml_hexagon_discover_devices() { // Populate domain IDs and names for all configured devices for (size_t i = 0; i < opt_ndev; i++) { - auto & cfg = opt_device_configs[i]; - if (discovery_supported) { - auto it = cdsp_map.find(cfg.physical_idx); - if (it != cdsp_map.end()) { - cfg.domain_id = it->second.id; - cfg.domain_name = it->second.name; - } else { - GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not found on device (%zu CDSP core(s) available)\n", - cfg.physical_idx, cdsp_map.size()); - cfg.domain_id = -1; - cfg.domain_name = ""; - } - } else { - switch (cfg.physical_idx) { - case 0: - cfg.domain_id = 3; - cfg.domain_name = CDSP_DOMAIN_NAME; - break; - case 1: - cfg.domain_id = 4; - cfg.domain_name = "cdsp1"; - break; - default: - GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not supported without dynamic discovery\n", - cfg.physical_idx); - cfg.domain_id = -1; - cfg.domain_name = ""; - break; - } - } + ggml_hexagon_resolve_device_domain(opt_device_configs[i], discovery_supported, cdsp_map); } } @@ -6316,6 +7748,10 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_IQ4_NL == (unsigned int) GGML_TYPE_IQ4_NL, "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q4_K == (unsigned int) GGML_TYPE_Q4_K, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q6_K == (unsigned int) GGML_TYPE_Q6_K, + "please update hexagon_type to match ggml_type"); const char * str_verbose = getenv("GGML_HEXAGON_VERBOSE"); const char * str_opbatch = getenv("GGML_HEXAGON_OPBATCH"); @@ -6329,6 +7765,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { const char * str_nhmx = getenv("GGML_HEXAGON_NHMX"); const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT"); const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT"); + const char * str_gdn_select = getenv("GGML_HEXAGON_GDN_SELECT"); const char * str_ar_select = getenv("GGML_HEXAGON_AR_SELECT"); const char * str_ndev = getenv("GGML_HEXAGON_NDEV"); const char * str_arch = getenv("GGML_HEXAGON_ARCH"); @@ -6336,6 +7773,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { const char * str_mbuf = getenv("GGML_HEXAGON_MBUF"); const char * str_optrace = getenv("GGML_HEXAGON_OPTRACE"); const char * str_hostbuf = getenv("GGML_HEXAGON_HOSTBUF"); + const char * str_dma64 = getenv("GGML_HEXAGON_DMA64"); // Init Arch first since it affects other defaults if (!str_arch) { @@ -6363,6 +7801,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { // Update vmem default opt_vmem = opt_arch >= 75 ? HTP_OP_MAX_VMEM_DEFAULT : 3000 * MiB; + opt_dma64 = opt_arch > 79 && (!str_dma64 || atoi(str_dma64) != 0); auto RE_ICASE = std::regex_constants::icase; @@ -6379,6 +7818,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_nhmx = str_nhmx ? atoi(str_nhmx) : opt_nhmx; opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select; opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select; + opt_gdn_select = str_gdn_select ? atoi(str_gdn_select) : opt_gdn_select; opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select; opt_mbuf = str_mbuf ? strtoul(str_mbuf, NULL, 0) * MiB : opt_mbuf; opt_vmem = str_vmem ? strtoul(str_vmem, NULL, 0) * MiB : opt_vmem; @@ -6408,21 +7848,126 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_device_configs[i].physical_idx = 0; opt_device_configs[i].virtual_idx = (int)i; opt_device_configs[i].name = "HTP" + std::to_string(i); + opt_device_configs[i].mdev_group.clear(); } } else { std::string s_devices(str_devices); - std::stringstream ss(s_devices); - std::string item; - opt_ndev = 0; - while (std::getline(ss, item, ',')) { - size_t start = item.find_first_not_of(" \t\r\n"); - size_t end = item.find_last_not_of(" \t\r\n"); - if (start == std::string::npos) { - continue; + std::vector<std::string> items; + std::string curr_item; + int bracket_depth = 0; + for (char ch : s_devices) { + if (ch == '[') { + bracket_depth++; + curr_item += ch; + } else if (ch == ']') { + if (bracket_depth > 0) bracket_depth--; + curr_item += ch; + } else if (ch == ',' && bracket_depth == 0) { + size_t s = curr_item.find_first_not_of(" \t\r\n"); + size_t e = curr_item.find_last_not_of(" \t\r\n"); + if (s != std::string::npos) { + items.push_back(curr_item.substr(s, e - s + 1)); + } + curr_item.clear(); + } else { + curr_item += ch; } - item = item.substr(start, end - start + 1); + } + size_t s = curr_item.find_first_not_of(" \t\r\n"); + size_t e = curr_item.find_last_not_of(" \t\r\n"); + if (s != std::string::npos) { + items.push_back(curr_item.substr(s, e - s + 1)); + } + + opt_ndev = 0; + for (const auto & item : items) { + size_t b_open = item.find('['); + size_t b_close = item.rfind(']'); + + if (b_open != std::string::npos && b_close != std::string::npos && b_close > b_open) { + // Grouped / composite syntax: Name[phys_spec:virt] or Name[phys_spec] + std::string dev_name = item.substr(0, b_open); + std::string content = item.substr(b_open + 1, b_close - b_open - 1); + + int virt = 0; + std::string phys_spec = content; + size_t colon_pos = content.find(':'); + if (colon_pos != std::string::npos) { + phys_spec = content.substr(0, colon_pos); + try { + virt = std::stoi(content.substr(colon_pos + 1)); + } catch (...) { + virt = 0; + } + } else { + size_t dev_colon = dev_name.find(':'); + if (dev_colon != std::string::npos) { + try { + virt = std::stoi(dev_name.substr(dev_colon + 1)); + } catch (...) { + virt = 0; + } + } + } + + // Parse physical indices from phys_spec (e.g. 0-1, 0,1, 0-3, etc.) + std::vector<int> phys_list; + std::stringstream pss(phys_spec); + std::string p_part; + while (std::getline(pss, p_part, ',')) { + size_t ps = p_part.find_first_not_of(" \t\r\n"); + size_t pe = p_part.find_last_not_of(" \t\r\n"); + if (ps == std::string::npos) continue; + p_part = p_part.substr(ps, pe - ps + 1); + + size_t dash_pos = p_part.find('-'); + if (dash_pos != std::string::npos) { + try { + int p_start = std::stoi(p_part.substr(0, dash_pos)); + int p_end = std::stoi(p_part.substr(dash_pos + 1)); + for (int p = p_start; p <= p_end; p++) { + if (std::find(phys_list.begin(), phys_list.end(), p) == phys_list.end()) { + phys_list.push_back(p); + } + } + } catch (...) { + GGML_LOG_WARN("ggml-hex: failed to parse physical range in '%s'\n", p_part.c_str()); + } + } else { + try { + int p = std::stoi(p_part); + if (std::find(phys_list.begin(), phys_list.end(), p) == phys_list.end()) { + phys_list.push_back(p); + } + } catch (...) { + GGML_LOG_WARN("ggml-hex: failed to parse physical index in '%s'\n", p_part.c_str()); + } + } + } + + if (phys_list.empty()) { + phys_list.push_back(0); + } - if (item.rfind("HTP", 0) == 0) { + if (opt_ndev < GGML_HEXAGON_MAX_SESSIONS) { + auto & cfg = opt_device_configs[opt_ndev]; + cfg.name = dev_name; + cfg.physical_idx = phys_list[0]; + cfg.virtual_idx = virt; + cfg.mdev_group.clear(); + + for (size_t k = 1; k < phys_list.size(); k++) { + ggml_hexagon_device_config sub_cfg; + sub_cfg.physical_idx = phys_list[k]; + sub_cfg.virtual_idx = virt; + sub_cfg.name = "HTP" + std::to_string(phys_list[k]) + ":" + std::to_string(virt); + cfg.mdev_group.push_back(sub_cfg); + } + opt_ndev++; + } else { + GGML_LOG_WARN("ggml-hex: max sessions limit reached (%d), ignoring device %s\n", GGML_HEXAGON_MAX_SESSIONS, item.c_str()); + } + } else if (item.rfind("HTP", 0) == 0) { std::string rest = item.substr(3); size_t colon_pos = rest.find(':'); int phys = 0; @@ -6446,6 +7991,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_device_configs[opt_ndev].name = colon_pos == std::string::npos ? "HTP" + std::to_string(phys) : "HTP" + std::to_string(phys) + ":" + std::to_string(virt); + opt_device_configs[opt_ndev].mdev_group.clear(); opt_ndev++; } else { GGML_LOG_WARN("ggml-hex: max sessions limit reached (%d), ignoring device %s\n", GGML_HEXAGON_MAX_SESSIONS, item.c_str()); @@ -6460,6 +8006,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_device_configs[0].physical_idx = 0; opt_device_configs[0].virtual_idx = 0; opt_device_configs[0].name = "HTP0"; + opt_device_configs[0].mdev_group.clear(); } #if defined(__ANDROID__) diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h index b083e26718bd..803aa3f5a4f1 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.h +++ b/ggml/src/ggml-hexagon/htp-opnode.h @@ -14,7 +14,11 @@ #include "htp/matmul-ops.h" #include "htp/flash-attn-ops.h" #include "htp/unary-ops.h" +#include "htp/binary-ops.h" #include "htp/allreduce-ops.h" +#include "htp/ssm-conv.h" +#include "htp/gated-delta-net-ops.h" +#include "htp/softmax-ops.h" struct htp_opnode { ggml_tensor * node { nullptr }; @@ -325,10 +329,6 @@ struct htp_opformat { } else if (type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || type == HTP_MM_KERNEL_HVX_F32_F32_VTCM || type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { path = "hvx-tiled"; - } else if (type == HTP_MM_KERNEL_HVX_F16_F16_DDR || type == HTP_MM_KERNEL_HVX_F16_F32_DDR || - type == HTP_MM_KERNEL_HVX_F32_F32_DDR || type == HTP_MM_KERNEL_HVX_F32_F16_DDR || - type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { - path = "hvx-flat"; } snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { @@ -344,6 +344,29 @@ struct htp_opformat { } else if (htp_op_is_unary(node.opcode)) { const auto * kparams = (const struct htp_unary_kernel_params *) node.kernel_params; snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) { + snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]); + } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) { + snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]); + } else if (node.opcode == HTP_OP_ALLREDUCE && node.node) { + snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]); + } else if (node.opcode == HTP_OP_SSM_CONV) { + const auto * kparams = (const struct htp_ssm_conv_kernel_params *) node.kernel_params; + snprintf(str, max_size, "%s vtcm %d", kparams->n_t == 1 ? "decode" : "prefill", (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_SOFTMAX) { + const auto * kparams = (const struct htp_softmax_kernel_params *) node.kernel_params; + snprintf(str, max_size, "k%d nth %d vtcm %d", (int) kparams->kernel_id, (int) kparams->n_threads, (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_GATED_DELTA_NET) { + const auto * kparams = (const struct htp_gdn_kernel_params *) node.kernel_params; + const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent"; + snprintf(str, max_size, "%s-%s vtcm %u", + path, + kparams->kda ? "kda" : "scalar", + (unsigned int) (kparams->vtcm_size ? kparams->vtcm_size : kparams->vtcm_per_thread * kparams->n_threads)); + } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || + node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { + const auto * kparams = (const struct htp_binary_kernel_params *) node.kernel_params; + snprintf(str, max_size, "vtcm %u", (unsigned int) kparams->vtcm_size); } else { snprintf(str, max_size, "----"); } diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index 77f3ee39dd3c..821f08c0bebf 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -43,6 +43,7 @@ add_library(${HTP_LIB} SHARED pad-ops.c argsort-ops.c im2col-ops.c + roll-ops.c allreduce-ops.c ) diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c index ac00b447d989..d59ac0770c2f 100644 --- a/ggml/src/ggml-hexagon/htp/act-ops.c +++ b/ggml/src/ggml-hexagon/htp/act-ops.c @@ -3,19 +3,18 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include <HAP_farf.h> -#include <HAP_perf.h> #include <math.h> #include <string.h> -#include "hex-dma.h" +#include "dma-queue.h" #include "hvx-utils.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "hex-common.h" #include "htp-tensor.h" #include "htp-vtcm.h" @@ -54,13 +53,24 @@ const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; +struct htp_act_context; + +typedef void (*glu_compute_fn_t)(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx); + struct htp_act_context { struct htp_ops_context * octx; + glu_compute_fn_t compute; + const char * op_str; + // Precomputed values - const uint8_t * data_src0; - const uint8_t * data_src1; - uint8_t * data_dst; + dma_addr_t data_src0; + dma_addr_t data_src1; + dma_addr_t data_dst; size_t src0_row_size; size_t src1_row_size; @@ -80,6 +90,7 @@ struct htp_act_context { uint32_t block; uint32_t src0_nrows; uint32_t src0_nrows_per_thread; + uint32_t row_start; int nc; uint8_t * vtcm_src0; @@ -134,10 +145,10 @@ static inline void htp_act_vtcm_layout_build(struct htp_act_vtcm_layout * L, // swiglu(x) = x1 * sigmoid(x0) static void swiglu_f32(const float * restrict src0, - const float * restrict src1, - float * restrict dst, - const uint32_t num_rows, - const struct htp_act_context * actx) { + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { htp_glu_op_preamble; for (uint32_t ib = 0; ib < num_rows; ib++) { @@ -152,10 +163,10 @@ static void swiglu_f32(const float * restrict src0, // out = x * sigmoid(alpha * x) * (clamp(y, -limit, limit) + 1.f) static void swiglu_oai_f32(const float * restrict src0, - const float * restrict src1, - float * restrict dst, - const uint32_t num_rows, - const struct htp_act_context * actx) { + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { htp_glu_op_preamble; const float alpha = ((const float *) (actx->octx->op_params))[2]; const float limit = ((const float *) (actx->octx->op_params))[3]; @@ -181,10 +192,10 @@ static void swiglu_oai_f32(const float * restrict src0, } static void swiglu_clamp_f32(const float * restrict src0, - const float * restrict src1, - float * restrict dst, - const uint32_t num_rows, - const struct htp_act_context * actx) { + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { htp_glu_op_preamble; const float limit = ((const float *) (actx->octx->op_params))[3]; @@ -312,12 +323,51 @@ static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * rest } } +static inline void hvx_geglu_quick_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + const HVX_Vector v_scale = hvx_vec_splat_f32(1.702f); + const HVX_Vector v_one = hvx_vec_splat_f32(1.0f); + const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f); + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + HVX_Vector scaled_x = hvx_vec_mul_f32_f32(x, v_scale); + HVX_Vector sigmoid_x = hvx_vec_fast_sigmoid_f32_guard_2it(scaled_x, v_one, v_max_exp, v_min_exp); + vdst[i] = hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(x, sigmoid_x), g); + } + + if (nloe) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + HVX_Vector scaled_x = hvx_vec_mul_f32_f32(x, v_scale); + HVX_Vector sigmoid_x = hvx_vec_fast_sigmoid_f32_guard_2it(scaled_x, v_one, v_max_exp, v_min_exp); + HVX_Vector result = hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(x, sigmoid_x), g); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), result); + } +} + // geglu(x, g) = gelu(x) * g static void geglu_f32(const float * restrict src0, - const float * restrict src1, - float * restrict dst, - const uint32_t num_rows, - const struct htp_act_context * actx) { + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { htp_glu_op_preamble; for (uint32_t ib = 0; ib < num_rows; ib++) { @@ -329,110 +379,117 @@ static void geglu_f32(const float * restrict src0, } } -#define DEFINE_GLU_PER_THREAD(NAME, OP_STR, CORE_EXPR) \ - static void glu_##NAME##_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_act_context * actx = (struct htp_act_context *) data; \ - htp_act_preamble; \ - \ - struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \ - \ - size_t src0_row_size = actx->src0_row_size; \ - size_t src1_row_size = actx->src1_row_size; \ - size_t dst_row_size = actx->dst_row_size; \ - \ - size_t src0_row_stride = actx->src0_row_stride; \ - size_t src1_row_stride = actx->src1_row_stride; \ - \ - const uint32_t src0_nrows = actx->src0_nrows; \ - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \ - \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ - \ - /* no work for this thread */ \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - const uint8_t * restrict data_src0 = actx->data_src0; \ - const uint8_t * restrict data_src1 = actx->data_src1; \ - uint8_t * restrict data_dst = actx->data_dst; \ - \ - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \ - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \ - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \ - \ - uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \ - uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \ - uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \ - \ - size_t src0_spad_half_size = actx->src0_spad_half_size; \ - size_t src1_spad_half_size = actx->src1_spad_half_size; \ - size_t dst_spad_half_size = actx->dst_spad_half_size; \ - \ - const int BLOCK = actx->block; \ - if (BLOCK == 0) { \ - FARF(ERROR, \ - OP_STR \ - " : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", \ - actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \ - return; \ - } \ - \ - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; \ - \ - /* See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 */ \ - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \ - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ - \ - /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \ - dma_queue_push_vtcm_to_ddr(dma_queue, \ - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \ - dst_row_size, dst_row_size_aligned, 0); \ - \ - dma_queue_push( \ - dma_queue, \ - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \ - src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \ - dma_queue_push( \ - dma_queue, \ - dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \ - src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \ - } \ - \ - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \ - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ - \ - float * dst_spad = (float *) dma_queue_pop(dma_queue).src; \ - float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; \ - float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ - CORE_EXPR; \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ - \ - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), \ - dst_row_size, dst_row_size_aligned, block_size); \ - \ - /* prefetch N+2 loop iteration if any */ \ - const uint32_t pref_block = (ir + BLOCK * 2); \ - if (pref_block < src0_end_row) { \ - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \ - dma_queue_push(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_stride)), \ - src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \ - dma_queue_push(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_stride)), \ - src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \ - } \ - } \ - \ - dma_queue_flush(dma_queue); \ - \ +// geglu_quick(x, g) = x * sigmoid(1.702 * x) * g +static void geglu_quick_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_geglu_quick_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +static void glu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { + struct htp_act_context * actx = (struct htp_act_context *) data; + htp_act_preamble; + + struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; + + size_t src0_row_size = actx->src0_row_size; + size_t src1_row_size = actx->src1_row_size; + size_t dst_row_size = actx->dst_row_size; + + size_t src0_row_stride = actx->src0_row_stride; + size_t src1_row_stride = actx->src1_row_stride; + + const uint32_t src0_nrows = actx->src0_nrows; + const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; + + const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); + + /* no work for this thread */ + if (src0_start_row >= src0_end_row) { + return; } -DEFINE_GLU_PER_THREAD(swiglu, "swiglu-f32", swiglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) -DEFINE_GLU_PER_THREAD(swiglu_oai, "swiglu-oai-f32", swiglu_oai_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) -DEFINE_GLU_PER_THREAD(swiglu_clamp, "swiglu-clamp-f32", swiglu_clamp_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) -DEFINE_GLU_PER_THREAD(geglu, "geglu-f32", geglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) + const dma_addr_t data_src0 = actx->data_src0; + const dma_addr_t data_src1 = actx->data_src1; + const dma_addr_t data_dst = actx->data_dst; + + const size_t src0_row_size_aligned = actx->src0_row_size_aligned; + const size_t src1_row_size_aligned = actx->src1_row_size_aligned; + const size_t dst_row_size_aligned = actx->dst_row_size_aligned; + + uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); + uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); + uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); + + size_t src0_spad_half_size = actx->src0_spad_half_size; + size_t src1_spad_half_size = actx->src1_spad_half_size; + size_t dst_spad_half_size = actx->dst_spad_half_size; + + const int BLOCK = actx->block; + if (BLOCK == 0) { + FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n", + actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned); + return; + } + + dma_queue * dma_q = actx->octx->ctx->dma[ith]; + glu_compute_fn_t compute = actx->compute; + + for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); + + /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ + dma_queue_push(dma_q, + dma_make_data(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), + dst_row_size, dst_row_size_aligned, dst_row_size, 0); + + dma_queue_push(dma_q, + dma_make_data(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), + src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); + + dma_queue_push(dma_q, + dma_make_data(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), + src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); + } + + for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); + + float * dst_spad = (float *) dma_queue_pop(dma_q).src; + float * src0_spad = (float *) dma_queue_pop(dma_q).dst; + float * src1_spad = (float *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); + compute(src0_spad, src1_spad, dst_spad, block_size, actx); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); + + dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad), + dst_row_size, dst_row_size_aligned, dst_row_size, block_size); + + /* prefetch N+2 loop iteration if any */ + const uint32_t pref_block = (ir + BLOCK * 2); + if (pref_block < src0_end_row) { + const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); + dma_queue_push(dma_q, dma_make_data(src0_spad, data_src0 + (pref_block * src0_row_stride)), + src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); + dma_queue_push(dma_q, dma_make_data(src1_spad, data_src1 + (pref_block * src1_row_stride)), + src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); + } + } + + dma_queue_flush(dma_q); +} static int execute_op_activations_f32(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; @@ -444,28 +501,33 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - worker_callback_t act_op_func; - const char * op_type = NULL; + glu_compute_fn_t compute_fn = NULL; + const char * op_type = NULL; switch (octx->op) { case HTP_OP_GLU_SWIGLU: - act_op_func = (worker_callback_t)glu_swiglu_f32_per_thread; - op_type = "swiglu-f32"; + compute_fn = swiglu_f32; + op_type = "swiglu-f32"; break; case HTP_OP_GLU_SWIGLU_OAI: - act_op_func = (worker_callback_t)glu_swiglu_oai_f32_per_thread; - op_type = "swiglu-oai-f32"; + compute_fn = swiglu_oai_f32; + op_type = "swiglu-oai-f32"; break; case HTP_OP_GLU_SWIGLU_CLAMP: - act_op_func = (worker_callback_t) glu_swiglu_clamp_f32_per_thread; - op_type = "swiglu-clamp-f32"; + compute_fn = swiglu_clamp_f32; + op_type = "swiglu-clamp-f32"; break; case HTP_OP_GLU_GEGLU: - act_op_func = (worker_callback_t)glu_geglu_f32_per_thread; - op_type = "geglu-f32"; + compute_fn = geglu_f32; + op_type = "geglu-f32"; + break; + + case HTP_OP_GLU_GEGLU_QUICK: + compute_fn = geglu_quick_f32; + op_type = "geglu-quick-f32"; break; default: FARF(ERROR, "Unsupported activations Op %u\n", octx->op); @@ -473,14 +535,30 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { } const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); + const size_t dst_row_size = dst->ne[0] * SIZEOF_FP32; + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). - const size_t nc_bytes = dst->ne[0] * SIZEOF_FP32; - const size_t src0_row_size = nc_bytes; - const size_t src1_row_size = nc_bytes; - const size_t dst_row_size = nc_bytes; + const size_t nc_bytes = dst_row_size; + const size_t src0_row_size = nc_bytes; + const size_t src1_row_size = nc_bytes; const size_t src0_row_stride = src0->nb[1]; const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; @@ -510,15 +588,13 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); } - if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - return HTP_STATUS_OK; - } - // Prepare context struct htp_act_context actx; - actx.octx = octx; + actx.octx = octx; + actx.compute = compute_fn; + actx.op_str = op_type; - actx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); actx.src0_row_size = src0_row_size; actx.src1_row_size = src1_row_size; @@ -545,15 +621,20 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { actx.dst_spad_half_size = L.dst_bytes_per_thread / 2; actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned; - actx.src0_nrows = src0_nrows; + actx.src0_nrows = nrows; + actx.row_start = row_start; actx.nc = dst->ne[0]; - // Pointers and GLU logic - const uint8_t * data_src0 = (const uint8_t *) src0->data; - const uint8_t * data_src1 = src1 ? (const uint8_t *) src1->data : NULL; + // Addresses and GLU logic + dma_addr_t data_src0 = src0->data; + dma_addr_t data_src1 = src1 ? src1->data : 0; - if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || octx->op == HTP_OP_GLU_SWIGLU_OAI || octx->op == HTP_OP_GLU_SWIGLU_CLAMP || octx->op == HTP_OP_GLU_GEGLU)) { + if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || + octx->op == HTP_OP_GLU_SWIGLU_OAI || + octx->op == HTP_OP_GLU_SWIGLU_CLAMP || + octx->op == HTP_OP_GLU_GEGLU || + octx->op == HTP_OP_GLU_GEGLU_QUICK)) { const int32_t swapped = octx->op_params[1]; data_src1 = data_src0; actx.src1_row_size = actx.src0_row_size; @@ -568,9 +649,9 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { actx.data_src0 = data_src0; actx.data_src1 = data_src1; - actx.data_dst = (uint8_t *) dst->data; + actx.data_dst = dst->data; - worker_pool_run_func(octx->ctx->worker_pool, act_op_func, &actx, n_threads); + work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f32_per_thread, &actx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.c b/ggml/src/ggml-hexagon/htp/allreduce-ops.c index d35f685a6dc0..7b577befbbc0 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.c +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.c @@ -14,9 +14,10 @@ #include "htp-ops.h" #include "hvx-utils.h" #include "htp-tensor.h" -#include "hex-dma.h" +#include "dma-queue.h" #include "hex-profile.h" #include "allreduce-ops.h" +#include "htp-fence.h" struct htp_allreduce_context { struct htp_ops_context * octx; @@ -37,97 +38,97 @@ struct htp_allreduce_context { uint8_t * res_spad_base; }; -#define DEFINE_ALLREDUCE_THREAD_DMA_1D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD) \ -static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ - struct htp_ops_context * octx = actx->octx; \ - \ - const uint32_t n_ranks = actx->n_ranks; \ - const uint32_t n_dsts = actx->n_dsts; \ - const uint32_t block_elems = actx->block_elems; \ - \ - const uint32_t dr = actx->elems_per_thread; \ - const uint32_t ir0 = actx->rank_elem_start + dr * ith; \ - const uint32_t ir1 = MIN(ir0 + dr, actx->rank_elem_start + actx->rank_nelem); \ - if (ir0 >= ir1) return; \ - \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - dma_queue * q = octx->ctx->dma[ith]; \ - \ - uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; \ - for (uint32_t s = 0; s < n_ranks; s++) { \ - src_spad_base[s] = actx->src_spad_base[s] + (ith * actx->vtcm_size_per_thread); \ - } \ - uint8_t * dst_spad_base = actx->dst_spad_base + (ith * actx->vtcm_size_per_thread); \ - uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + (ith * actx->vtcm_size_per_thread)) : NULL; \ - \ - const size_t spad_half = actx->vtcm_size_per_thread / 2; \ - uint32_t ir_prefetch = ir0; \ - int spad_idx = 0; \ - \ - for (int k = 0; k < 2 && ir_prefetch < ir1; k++) { \ - uint32_t cur_elems = MIN(block_elems, ir1 - ir_prefetch); \ - size_t cur_bytes = cur_elems * sizeof(TYPE); \ - uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \ - for (uint32_t d = 0; d < n_dsts; d++) { \ - uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + ir_prefetch * sizeof(TYPE); \ - dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 0); \ - } \ - for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = src_spad_base[s] + spad_idx * spad_half; \ - const uint8_t * s_ddr = (const uint8_t *) octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ - dma_queue_push(q, dma_make_ptr(s_spad, s_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \ - } \ - if (HAS_ADD) { \ - uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \ - const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \ - dma_queue_push(q, dma_make_ptr(r_spad, r_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \ - } \ - ir_prefetch += cur_elems; \ - spad_idx ^= 1; \ - } \ - \ - for (uint32_t ir = ir0; ir < ir1; ) { \ - uint32_t cur_elems = MIN(block_elems, ir1 - ir); \ - size_t cur_bytes = cur_elems * sizeof(TYPE); \ - uint8_t * d_spad = NULL; \ - for (uint32_t d = 0; d < n_dsts; d++) { \ - d_spad = (uint8_t *) dma_queue_pop(q).src; \ - } \ - uint8_t * s_spad[HTP_ALLREDUCE_MAX_RANKS]; \ - for (uint32_t s = 0; s < n_ranks; s++) { \ - s_spad[s] = (uint8_t *) dma_queue_pop(q).dst; \ - } \ - uint8_t * r_spad = HAS_ADD ? (uint8_t *) dma_queue_pop(q).dst : NULL; \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \ - HVX_ADD_FN(d_spad, s_spad[0], s_spad[1], cur_elems); \ - for (uint32_t s = 2; s < n_ranks; s++) { \ - HVX_ADD_FN(d_spad, d_spad, s_spad[s], cur_elems); \ - } \ - if (HAS_ADD) { \ - HVX_ADD_FN(d_spad, d_spad, r_spad, cur_elems); \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \ - for (uint32_t d = 0; d < n_dsts; d++) { \ - uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + ir * sizeof(TYPE); \ - dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 1); \ - } \ - if (ir_prefetch < ir1) { \ - uint32_t next_elems = MIN(block_elems, ir1 - ir_prefetch); \ - size_t next_bytes = next_elems * sizeof(TYPE); \ - for (uint32_t s = 0; s < n_ranks; s++) { \ - const uint8_t * s_next = (const uint8_t *) octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ - dma_queue_push(q, dma_make_ptr(s_spad[s], s_next), next_bytes, next_bytes, next_bytes, 1); \ - } \ - if (HAS_ADD) { \ - const uint8_t * r_next = (const uint8_t *) octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \ - dma_queue_push(q, dma_make_ptr(r_spad, r_next), next_bytes, next_bytes, next_bytes, 1); \ - } \ - ir_prefetch += next_elems; \ - } \ - ir += cur_elems; \ - } \ - dma_queue_flush(q); \ +#define DEFINE_ALLREDUCE_THREAD_DMA_1D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD) \ +static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ + struct htp_ops_context * octx = actx->octx; \ + \ + const uint32_t n_ranks = actx->n_ranks; \ + const uint32_t n_dsts = actx->n_dsts; \ + const uint32_t block_elems = actx->block_elems; \ + \ + const uint32_t dr = actx->elems_per_thread; \ + const uint32_t ir0 = actx->rank_elem_start + dr * ith; \ + const uint32_t ir1 = MIN(ir0 + dr, actx->rank_elem_start + actx->rank_nelem); \ + if (ir0 >= ir1) return; \ + \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + \ + uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + src_spad_base[s] = actx->src_spad_base[s] + (ith * actx->vtcm_size_per_thread); \ + } \ + uint8_t * dst_spad_base = actx->dst_spad_base + (ith * actx->vtcm_size_per_thread); \ + uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + (ith * actx->vtcm_size_per_thread)) : NULL; \ + \ + const size_t spad_half = actx->vtcm_size_per_thread / 2; \ + uint32_t ir_prefetch = ir0; \ + int spad_idx = 0; \ + \ + for (int k = 0; k < 2 && ir_prefetch < ir1; k++) { \ + uint32_t cur_elems = MIN(block_elems, ir1 - ir_prefetch); \ + size_t cur_bytes = cur_elems * sizeof(TYPE); \ + uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + dma_addr_t d_ddr = octx->dsts[d]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 0); \ + } \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + uint8_t * s_spad = src_spad_base[s] + spad_idx * spad_half; \ + const dma_addr_t s_ddr = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \ + } \ + if (HAS_ADD) { \ + uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \ + const dma_addr_t r_ddr = octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(r_spad, r_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \ + } \ + ir_prefetch += cur_elems; \ + spad_idx ^= 1; \ + } \ + \ + for (uint32_t ir = ir0; ir < ir1; ) { \ + uint32_t cur_elems = MIN(block_elems, ir1 - ir); \ + size_t cur_bytes = cur_elems * sizeof(TYPE); \ + uint8_t * d_spad = NULL; \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + d_spad = (uint8_t *) dma_queue_pop(dma_q).src; \ + } \ + uint8_t * s_spad[HTP_ALLREDUCE_MAX_RANKS]; \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + s_spad[s] = (uint8_t *) dma_queue_pop(dma_q).dst; \ + } \ + uint8_t * r_spad = HAS_ADD ? (uint8_t *) dma_queue_pop(dma_q).dst : NULL; \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \ + HVX_ADD_FN(d_spad, s_spad[0], s_spad[1], cur_elems); \ + for (uint32_t s = 2; s < n_ranks; s++) { \ + HVX_ADD_FN(d_spad, d_spad, s_spad[s], cur_elems); \ + } \ + if (HAS_ADD) { \ + HVX_ADD_FN(d_spad, d_spad, r_spad, cur_elems); \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + dma_addr_t d_ddr = octx->dsts[d]->data + ir * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 1); \ + } \ + if (ir_prefetch < ir1) { \ + uint32_t next_elems = MIN(block_elems, ir1 - ir_prefetch); \ + size_t next_bytes = next_elems * sizeof(TYPE); \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + const dma_addr_t s_next = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(s_spad[s], s_next), next_bytes, next_bytes, next_bytes, 1); \ + } \ + if (HAS_ADD) { \ + const dma_addr_t r_next = octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(r_spad, r_next), next_bytes, next_bytes, next_bytes, 1); \ + } \ + ir_prefetch += next_elems; \ + } \ + ir += cur_elems; \ + } \ + dma_queue_flush(dma_q); \ } DEFINE_ALLREDUCE_THREAD_DMA_1D(f16, __fp16, hvx_add_f16_aaa, 0) @@ -153,7 +154,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, if (r0 >= r1) return; \ \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - dma_queue * q = octx->ctx->dma[ith]; \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ \ uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; \ for (uint32_t s = 0; s < n_ranks; s++) { \ @@ -170,18 +171,18 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, uint32_t cur_rows = MIN(block_rows, r1 - r_prefetch); \ uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \ for (uint32_t d = 0; d < n_dsts; d++) { \ - uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + r_prefetch * octx->dsts[d]->nb[1]; \ - dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, 0); \ + dma_addr_t d_ddr = octx->dsts[d]->data + r_prefetch * octx->dsts[d]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, 0); \ } \ for (uint32_t s = 0; s < n_ranks; s++) { \ uint8_t * s_spad = src_spad_base[s] + spad_idx * spad_half; \ - const uint8_t * s_ddr = (const uint8_t *) octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ - dma_queue_push(q, dma_make_ptr(s_spad, s_ddr), row_size_aligned, octx->src[s]->nb[1], row_bytes, cur_rows); \ + const dma_addr_t s_ddr = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), row_size_aligned, octx->src[s]->nb[1], row_bytes, cur_rows); \ } \ if (HAS_ADD && !IS_ROW_BCAST) { \ uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \ - const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \ - dma_queue_push(q, dma_make_ptr(r_spad, r_ddr), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, cur_rows); \ + const dma_addr_t r_ddr = octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(r_spad, r_ddr), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, cur_rows); \ } \ r_prefetch += cur_rows; \ spad_idx ^= 1; \ @@ -191,13 +192,13 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, uint32_t cur_rows = MIN(block_rows, r1 - r); \ uint8_t * d_spad = NULL; \ for (uint32_t d = 0; d < n_dsts; d++) { \ - d_spad = (uint8_t *) dma_queue_pop(q).src; \ + d_spad = (uint8_t *) dma_queue_pop(dma_q).src; \ } \ uint8_t * s_spad[HTP_ALLREDUCE_MAX_RANKS]; \ for (uint32_t s = 0; s < n_ranks; s++) { \ - s_spad[s] = (uint8_t *) dma_queue_pop(q).dst; \ + s_spad[s] = (uint8_t *) dma_queue_pop(dma_q).dst; \ } \ - uint8_t * r_spad = (HAS_ADD && !IS_ROW_BCAST) ? (uint8_t *) dma_queue_pop(q).dst : NULL; \ + uint8_t * r_spad = (HAS_ADD && !IS_ROW_BCAST) ? (uint8_t *) dma_queue_pop(dma_q).dst : NULL; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \ for (uint32_t row = 0; row < cur_rows; row++) { \ uint8_t * d_row = d_spad + row * row_size_aligned; \ @@ -215,24 +216,24 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, } \ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \ for (uint32_t d = 0; d < n_dsts; d++) { \ - uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + r * octx->dsts[d]->nb[1]; \ - dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, cur_rows); \ + dma_addr_t d_ddr = octx->dsts[d]->data + r * octx->dsts[d]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, cur_rows); \ } \ if (r_prefetch < r1) { \ uint32_t next_rows = MIN(block_rows, r1 - r_prefetch); \ for (uint32_t s = 0; s < n_ranks; s++) { \ - const uint8_t * s_next = (const uint8_t *) octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ - dma_queue_push(q, dma_make_ptr(s_spad[s], s_next), row_size_aligned, octx->src[s]->nb[1], row_bytes, next_rows); \ + const dma_addr_t s_next = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(s_spad[s], s_next), row_size_aligned, octx->src[s]->nb[1], row_bytes, next_rows); \ } \ if (HAS_ADD && !IS_ROW_BCAST) { \ - const uint8_t * r_next = (const uint8_t *) octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \ - dma_queue_push(q, dma_make_ptr(r_spad, r_next), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, next_rows); \ + const dma_addr_t r_next = octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(r_spad, r_next), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, next_rows); \ } \ r_prefetch += next_rows; \ } \ r += cur_rows; \ } \ - dma_queue_flush(q); \ + dma_queue_flush(dma_q); \ } DEFINE_ALLREDUCE_THREAD_DMA_2D(f16, __fp16, hvx_add_f16_aaa, 0, 0) @@ -242,7 +243,42 @@ DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f32, float, hvx_add_f32_aaa, 1, 0) DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f16, __fp16, hvx_add_f16_aaa, 1, 1) DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f32, float, hvx_add_f32_aaa, 1, 1) +static int validate_allreduce( + struct htp_ops_context * octx, + const struct htp_allreduce_kernel_params * kparams, + uint32_t n_ranks +) { + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (kparams->vtcm_size_per_thread <= 0 || kparams->vtcm_size <= 0) { + return HTP_STATUS_INVAL_PARAMS; + } + + const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count( + n_ranks, octx->n_threads, has_add, kparams->is_row_bcast != 0); + const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_thread; + if (vtcm_size != (size_t) kparams->vtcm_size) { + return HTP_STATUS_INVAL_PARAMS; + } + if (vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + if (octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_F32) { + return HTP_STATUS_NO_SUPPORT; + } + + return HTP_STATUS_OK; +} + int op_allreduce(struct htp_ops_context * octx) { + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + const struct htp_allreduce_kernel_params * kparams = (const struct htp_allreduce_kernel_params *) octx->kernel_params; const struct htp_tensor * dst = octx->dst; @@ -253,38 +289,53 @@ int op_allreduce(struct htp_ops_context * octx) { return HTP_STATUS_INVAL_PARAMS; } - if (dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_F32) { - return HTP_STATUS_NO_SUPPORT; + const uint32_t fence_seq_entry = (uint32_t) octx->op_params[0]; + const uint32_t fence_seq_exit = (uint32_t) octx->op_params[1]; + + const struct htp_tensor * my_sync = octx->src[n_ranks + rank]; + atomic_uint * my_fence = (atomic_uint *) (uintptr_t) my_sync->data; + + const int status = validate_allreduce(octx, kparams, n_ranks); + if (status != HTP_STATUS_OK) { + if (status == HTP_STATUS_NO_SUPPORT) { + FARF(ERROR, "ggml-hex: allreduce unsupported type %d : rank %u\n", dst->type, rank); + } + htp_fence_write(my_fence, fence_seq_exit, status); + return status; } + const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); const uint32_t nelem = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; - const uint32_t fence_seq_entry = (uint32_t) octx->op_params[0]; - const uint32_t fence_seq_exit = (uint32_t) octx->op_params[1]; // 1. Entry Barrier: Synchronize all ranks before reading struct htp_thread_trace * tr0 = &octx->ctx->trace[0]; htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); - const struct htp_tensor * my_sync = octx->src[n_ranks + rank]; - atomic_uint * my_fence = (atomic_uint *) my_sync->data; - - atomic_store(&my_fence[0], fence_seq_entry); - asm volatile ("syncht" : : : "memory"); - Q6_dccleaninva_A((void *) my_fence); + htp_fence_write(my_fence, fence_seq_entry, octx->status); for (uint32_t j = 0; j < n_ranks; j++) { if (j == rank) continue; const struct htp_tensor * peer_sync = octx->src[n_ranks + j]; - atomic_uint * peer_fence = (atomic_uint *) peer_sync->data; + atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data; uint64_t spins = 0; while (1) { - Q6_dccleaninva_A((void *) peer_fence); - uint32_t val = atomic_load(&peer_fence[0]); - if (val == fence_seq_entry || val == fence_seq_exit) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - fence_seq_entry) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: allreduce entry peer %u failed with status %u\n", j, peer_status); + htp_fence_write(my_fence, fence_seq_exit, peer_status); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); + return peer_status; + } break; } if (++spins > HTP_FENCE_TIMEOUT) { - FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_entry); + FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n", + rank, j, peer_fence, fence_seq_entry, peer_seq); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); return HTP_STATUS_INTERNAL_ERR; } hex_pause(); @@ -301,8 +352,6 @@ int op_allreduce(struct htp_ops_context * octx) { const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread; const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread; - const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); - struct htp_allreduce_context actx; actx.octx = octx; actx.n_ranks = n_ranks; @@ -339,6 +388,8 @@ int op_allreduce(struct htp_ops_context * octx) { } break; default: + FARF(ERROR, "ggml-hex: allreduce unsupported kernel %d : rank %u\n", kparams->kernel_type, rank); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_NO_SUPPORT); return HTP_STATUS_NO_SUPPORT; } @@ -355,11 +406,11 @@ int op_allreduce(struct htp_ops_context * octx) { } if (has_add && actx.is_row_bcast) { - const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data; + const dma_addr_t r_ddr = octx->src[2 * n_ranks]->data; const uint32_t row_bytes = actx.ne0 * (dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float)); - dma_queue * q = octx->ctx->dma[0]; - dma_queue_push(q, dma_make_ptr(actx.res_spad_base, r_ddr), actx.row_size_aligned, 0, row_bytes, 1); - dma_queue_pop(q); + dma_queue * dma_q = octx->ctx->dma[0]; + dma_queue_push(dma_q, dma_make_data(actx.res_spad_base, r_ddr), actx.row_size_aligned, 0, row_bytes, 1); + dma_queue_pop(dma_q); } work_queue_run(octx->ctx->work_queue, reduce_fun, &actx, n_threads); @@ -368,23 +419,31 @@ int op_allreduce(struct htp_ops_context * octx) { // 4. Exit Barrier: Synchronize all ranks after writing htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); - atomic_store(&my_fence[0], fence_seq_exit); - asm volatile ("syncht" : : : "memory"); - Q6_dccleaninva_A((void *) my_fence); + htp_fence_write(my_fence, fence_seq_exit, octx->status); for (uint32_t j = 0; j < n_ranks; j++) { if (j == rank) continue; const struct htp_tensor * peer_sync = octx->src[n_ranks + j]; - atomic_uint * peer_fence = (atomic_uint *) peer_sync->data; + atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data; uint64_t spins = 0; while (1) { - Q6_dccleaninva_A((void *) peer_fence); - uint32_t val = atomic_load(&peer_fence[0]); - if (val == fence_seq_exit) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - fence_seq_exit) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: allreduce exit peer %u failed with status %u\n", j, peer_status); + htp_fence_write(my_fence, fence_seq_exit, peer_status); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); + return peer_status; + } break; } if (++spins > HTP_FENCE_TIMEOUT) { - FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_exit); + FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n", + rank, j, peer_fence, fence_seq_exit, peer_seq); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); return HTP_STATUS_INTERNAL_ERR; } hex_pause(); @@ -394,5 +453,5 @@ int op_allreduce(struct htp_ops_context * octx) { htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); - return HTP_STATUS_OK; + return octx->status; } diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.h index de447d87e912..0aed2b8b7e67 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.h +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h @@ -2,6 +2,8 @@ #define ALLREDUCE_OPS_H #include <stdint.h> +#include <stddef.h> +#include <stdbool.h> #define HTP_ALLREDUCE_MAX_RANKS 4 @@ -15,6 +17,15 @@ enum htp_allreduce_kernel_type { HTP_ALLREDUCE_KERNEL_DMA_2D, }; +static inline size_t htp_allreduce_vtcm_buffer_count( + uint32_t n_ranks, + uint32_t n_threads, + bool has_add, + bool is_row_bcast +) { + return (size_t) (n_ranks + 1) * n_threads + (has_add ? (is_row_bcast ? 1 : n_threads) : 0); +} + struct htp_allreduce_kernel_params { int32_t rank; int32_t n_ranks; diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.c index 774faef5f388..9e9e74651612 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.c +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c @@ -9,11 +9,12 @@ #include "ggml.h" #include "hvx-utils.h" -#include "hex-dma.h" +#include "dma-queue.h" +#include "hex-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) @@ -22,6 +23,9 @@ struct htp_argsort_context { struct htp_ops_context * octx; uint32_t nrows_per_thread; + uint32_t total_rows; + uint32_t row_start; + uint32_t row_end; uint8_t * vtcm_base; size_t vtcm_per_thread; }; @@ -166,6 +170,21 @@ static void quicksort_values_indices_desc(float * values, int32_t * indices, int if (i < right) quicksort_values_indices_desc(values, indices, i, right); } +static uint32_t top_k_max_value_index(const float * values, uint32_t n, float * value) { + uint32_t index = 0; + float max_value = values[0]; + + for (uint32_t i = 1; i < n; i++) { + if (values[i] > max_value) { + max_value = values[i]; + index = i; + } + } + + *value = max_value; + return index; +} + // LUT for ramp initialization of argsort output (first 32 members) int32_t argosrt_ramp_lut[32] __attribute__((aligned(VLEN))) = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, @@ -299,6 +318,129 @@ static inline void bitonic_sort_generic_hvx(uint8_t * values, uint8_t * indices, } } + +// Sorts descending; values pre-padded with -INFINITY. init_indices=true resets +// indices to a fresh ramp (normal full-row sort); false leaves caller-supplied +// indices in place and only permutes them (used when merging candidates, to +// preserve their original global index). + +static void bitonic_sort_vtcm_desc(uint8_t * values, uint8_t * indices, uint32_t n_vec, bool init_indices) { + HVX_Vector zero_vec = Q6_V_vzero(); + HVX_Vector idx_vec = *(HVX_Vector *)argosrt_ramp_lut; + + HVX_VectorPred pred_all_1s = Q6_Q_vcmp_eq_VwVw(zero_vec, zero_vec); + HVX_VectorPred pred_all_0s = Q6_Q_not_Q(pred_all_1s); + + if (init_indices) { + // Initialize indices ramp (values are already populated by the caller) + for (uint32_t v = 0; v < n_vec; v++) { + HVX_Vector idx = Q6_Vw_vadd_VwVw(idx_vec, Q6_V_vsplat_R(v * 32)); + *(HVX_Vector *)(indices + v * 128) = idx; + } + } + + int M = 5; + while ((1u << (M - 5)) < n_vec) M++; + + for (int s = 1; s <= M; s++) { + for (int stage_d = s - 1; stage_d >= 0; stage_d--) { + int d = 1 << stage_d; + if (d >= 32) { + uint32_t v_dist = d / 32; + for (uint32_t v1 = 0; v1 < n_vec; v1++) { + if ((v1 & v_dist) == 0) { + uint32_t v2 = v1 + v_dist; + bool asc = (s < M) ? ((((v1 * 32) >> s) % 2) == 0) : false; + + HVX_Vector Vv1 = *(HVX_Vector *)(values + v1 * 128); + HVX_Vector Iv1 = *(HVX_Vector *)(indices + v1 * 128); + HVX_Vector Vv2 = *(HVX_Vector *)(values + v2 * 128); + HVX_Vector Iv2 = *(HVX_Vector *)(indices + v2 * 128); + + vec_cas(&Vv1, &Iv1, &Vv2, &Iv2, asc); + + *(HVX_Vector *)(values + v1 * 128) = Vv1; + *(HVX_Vector *)(indices + v1 * 128) = Iv1; + *(HVX_Vector *)(values + v2 * 128) = Vv2; + *(HVX_Vector *)(indices + v2 * 128) = Iv2; + } + } + } else { + if (s < 5) { + HVX_VectorPred dir_mask = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(idx_vec, Q6_V_vsplat_R(1 << s)), zero_vec); + for (uint32_t v = 0; v < n_vec; v++) { + HVX_Vector Vv = *(HVX_Vector *)(values + v * 128); + HVX_Vector Iv = *(HVX_Vector *)(indices + v * 128); + + bitonic_cas_32(&Vv, &Iv, d, dir_mask, idx_vec, zero_vec); + + *(HVX_Vector *)(values + v * 128) = Vv; + *(HVX_Vector *)(indices + v * 128) = Iv; + } + } else { + for (uint32_t v = 0; v < n_vec; v++) { + bool asc = (s < M) ? ((((v * 32) >> s) % 2) == 0) : false; + HVX_VectorPred dir_mask = asc ? pred_all_1s : pred_all_0s; + + HVX_Vector Vv = *(HVX_Vector *)(values + v * 128); + HVX_Vector Iv = *(HVX_Vector *)(indices + v * 128); + + bitonic_cas_32(&Vv, &Iv, d, dir_mask, idx_vec, zero_vec); + + *(HVX_Vector *)(values + v * 128) = Vv; + *(HVX_Vector *)(indices + v * 128) = Iv; + } + } + } + } + } +} + +static void top_k_select_tiled(const uint8_t * src, uint32_t n, uint32_t k, + float * values_buf, int32_t * indices_buf, + float * top_values, int32_t * top_indices) { + const uint32_t tile_elems = 1024; + uint32_t n_tiles = (n + tile_elems - 1) / tile_elems; + uint32_t candidate_count = n_tiles * k; + uint32_t merge_n_vec = hmx_ceil_div(candidate_count, 32); + uint32_t merge_n_vec_pow2 = 1; + while (merge_n_vec_pow2 < merge_n_vec) merge_n_vec_pow2 <<= 1; + uint32_t merge_elems = merge_n_vec_pow2 * 32; + float * candidate_values = values_buf + tile_elems; + int32_t * candidate_indices = indices_buf + tile_elems; + uint32_t candidate_pos = 0; + + for (uint32_t offset = 0; offset < n; offset += tile_elems) { + uint32_t tile_count = MIN(tile_elems, n - offset); + hvx_copy_f32_au((uint8_t *) values_buf, src + offset * sizeof(float), tile_count); + if (tile_count < tile_elems) { + hvx_splat_f32_u((uint8_t *) (values_buf + tile_count), -INFINITY, tile_elems - tile_count); + } + + bitonic_sort_vtcm_desc((uint8_t *) values_buf, (uint8_t *) indices_buf, tile_elems / 32, true); + uint32_t tile_k = MIN(k, tile_count); + for (uint32_t j = 0; j < tile_k; j++) { + candidate_values[candidate_pos] = values_buf[j]; + candidate_indices[candidate_pos] = indices_buf[j] + (int32_t) offset; + candidate_pos++; + } + } + + if (merge_elems > candidate_pos) { + hvx_splat_f32_u((uint8_t *) (candidate_values + candidate_pos), -INFINITY, merge_elems - candidate_pos); + for (uint32_t j = candidate_pos; j < merge_elems; j++) { + candidate_indices[j] = 0; + } + } + + bitonic_sort_vtcm_desc((uint8_t *) candidate_values, (uint8_t *) candidate_indices, merge_n_vec_pow2, false); + + for (uint32_t j = 0; j < k; j++) { + top_values[j] = candidate_values[j]; + top_indices[j] = candidate_indices[j]; + } +} + __attribute__((always_inline)) static inline void sort32_f32_hvx(uint8_t * values, uint8_t * indices, enum ggml_sort_order order) { bitonic_sort_generic_hvx(values, indices, 1, order == GGML_SORT_ORDER_ASC); @@ -336,10 +478,9 @@ static void htp_argsort_f32_##ne00##_##order_name(unsigned int n, unsigned int i const struct htp_tensor * src0 = octx->src[0]; \ const struct htp_tensor * dst = octx->dst; \ uint8_t * spad = actx->vtcm_base + actx->vtcm_per_thread * i; \ - uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; \ uint32_t rows_per_thread = actx->nrows_per_thread; \ - uint32_t start_row = rows_per_thread * i; \ - uint32_t end_row = MIN(start_row + rows_per_thread, total_rows); \ + uint32_t start_row = actx->row_start + rows_per_thread * i; \ + uint32_t end_row = MIN(start_row + rows_per_thread, actx->row_end); \ size_t values_size = hex_round_up(ne00 * sizeof(float), 128); \ float * values_buf = (float *) spad; \ int32_t * indices_buf = (int32_t *) (spad + values_size); \ @@ -386,9 +527,6 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * data // Dimensions uint32_t ne00 = src0->ne[0]; - uint32_t ne01 = src0->ne[1]; - uint32_t ne02 = src0->ne[2]; - uint32_t ne03 = src0->ne[3]; uint32_t nb01 = src0->nb[1]; @@ -398,10 +536,9 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * data enum ggml_sort_order order = (enum ggml_sort_order) octx->op_params[0]; // Rows to process - uint32_t total_rows = ne01 * ne02 * ne03; uint32_t rows_per_thread = actx->nrows_per_thread; - uint32_t start_row = rows_per_thread * i; - uint32_t end_row = MIN(start_row + rows_per_thread, total_rows); + uint32_t start_row = actx->row_start + rows_per_thread * i; + uint32_t end_row = MIN(start_row + rows_per_thread, actx->row_end); size_t values_size = hex_round_up(ne00 * sizeof(float), 128); uint32_t num_vec_ind_values = hmx_ceil_div(ne00, VLEN/(sizeof(int32_t))); @@ -451,8 +588,32 @@ int op_argsort(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t total_rows = octx->src[0]->ne[1] * octx->src[0]->ne[2] * octx->src[0]->ne[3]; - const uint32_t n_threads = MIN(total_rows, octx->n_threads); + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_row_size = dst->ne[0] * sizeof(int32_t); + + uint32_t row_start = 0; + uint32_t row_end = total_rows; + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(int32_t), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + row_end = range.start + range.count; + } + + const uint32_t nrows = row_end - row_start; + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; // Allocate scratchpad // We need 1 row of float + 1 row of int32 per thread. @@ -478,7 +639,10 @@ int op_argsort(struct htp_ops_context * octx) { struct htp_argsort_context actx; actx.octx = octx; - actx.nrows_per_thread = (total_rows + n_threads - 1) / n_threads; + actx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + actx.total_rows = nrows; + actx.row_start = row_start; + actx.row_end = row_end; actx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; actx.vtcm_per_thread = spad_per_thread; @@ -508,7 +672,424 @@ int op_argsort(struct htp_ops_context * octx) { } // Run jobs - worker_pool_run_func(octx->ctx->worker_pool, job_func, &actx, n_threads); + work_queue_run(octx->ctx->work_queue, job_func, &actx, n_threads); + + return HTP_STATUS_OK; +} + +// ggml_compute_forward_top_k +// +// Reuses ARGSORT's sort kernels. Only the first `k` indices are copied +// to dst, and there's no asc/desc param -- always largest-first. + +struct htp_top_k_context { + struct htp_ops_context * octx; + uint32_t nrows_per_thread; + uint32_t row_start; + uint32_t row_end; + uint8_t * vtcm_base; + size_t vtcm_per_thread; + uint32_t k; +}; + +#define HTP_TOP_K_FN(ne00, sort_fn) \ +static void htp_top_k_f32_##ne00(unsigned int n, unsigned int i, void * data) { \ + struct htp_top_k_context * actx = (struct htp_top_k_context *)data; \ + struct htp_ops_context * octx = actx->octx; \ + const struct htp_tensor * src0 = octx->src[0]; \ + const struct htp_tensor * dst = octx->dst; \ + uint8_t * spad = actx->vtcm_base + actx->vtcm_per_thread * i; \ + uint32_t row_start = actx->row_start; \ + uint32_t row_end = actx->row_end; \ + uint32_t rows_per_thread = actx->nrows_per_thread; \ + uint32_t start_row = row_start + rows_per_thread * i; \ + uint32_t end_row = MIN(start_row + rows_per_thread, row_end); \ + size_t values_size = hex_round_up(ne00 * sizeof(float), 128); \ + float * values_buf = (float *) spad; \ + int32_t * indices_buf = (int32_t *) (spad + values_size); \ + uint32_t nb01 = src0->nb[1]; \ + uint32_t nb1 = dst->nb[1]; \ + uint32_t k = actx->k; \ + struct htp_thread_trace * tr = &octx->ctx->trace[i]; \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start_row); \ + for (uint32_t r = start_row; r < end_row; r++) { \ + uint32_t src_offset = r * nb01; \ + uint32_t dst_offset = r * nb1; \ + uint8_t * src_ptr = (uint8_t *) src0->data + src_offset; \ + uint8_t * dst_ptr = (uint8_t *) dst->data + dst_offset; \ + hex_l2fetch(src_ptr, ne00 * sizeof(float), ne00 * sizeof(float), 1); \ + hvx_copy_f32_au((uint8_t*)values_buf, src_ptr, ne00); \ + sort_fn((uint8_t*)values_buf, (uint8_t*)indices_buf, GGML_SORT_ORDER_DESC); \ + hvx_copy_f32_ua(dst_ptr, (const uint8_t *) indices_buf, k); \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, start_row); \ +} + +HTP_TOP_K_FN(32, sort32_f32_hvx) +HTP_TOP_K_FN(64, sort64_f32_hvx) +HTP_TOP_K_FN(128, sort128_f32_hvx) +HTP_TOP_K_FN(256, sort256_f32_hvx) +HTP_TOP_K_FN(512, sort512_f32_hvx) +HTP_TOP_K_FN(1024, sort1024_f32_hvx) + +static void htp_top_k_f32_fallback(unsigned int n, unsigned int i, void * data) { + struct htp_top_k_context * actx = (struct htp_top_k_context *)data; + struct htp_ops_context * octx = actx->octx; + + // Unpack context + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + // Scratchpad memory + uint8_t * spad = actx->vtcm_base + actx->vtcm_per_thread * i; + + // Dimensions + uint32_t ne00 = src0->ne[0]; + uint32_t nb01 = src0->nb[1]; + + uint32_t nb1 = dst->nb[1]; + + uint32_t k = actx->k; + + // Rows to process + uint32_t row_start = actx->row_start; + uint32_t row_end = actx->row_end; + uint32_t rows_per_thread = actx->nrows_per_thread; + uint32_t start_row = row_start + rows_per_thread * i; + uint32_t end_row = MIN(start_row + rows_per_thread, row_end); + + // Pad ne00 to n_vec*32 (n_vec a power of 2) for the bitonic network; + // pad with -INFINITY so it never lands in the top-k. + uint32_t n_vec = hmx_ceil_div(ne00, 32); + uint32_t n_vec_pow2 = 1; + while (n_vec_pow2 < n_vec) n_vec_pow2 <<= 1; + uint32_t ne00_padded = n_vec_pow2 * 32; + + size_t values_size = hex_round_up(ne00_padded * sizeof(float), 128); + float * values_buf = (float *) spad; + int32_t * indices_buf = (int32_t *) (spad + values_size); + + struct htp_thread_trace * tr = &octx->ctx->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start_row); + + for (uint32_t r = start_row; r < end_row; r++) { + uint32_t src_offset = r * nb01; + uint32_t dst_offset = r * nb1; + + uint8_t * src_ptr = (uint8_t *) src0->data + src_offset; + uint8_t * dst_ptr = (uint8_t *) dst->data + dst_offset; + + hex_l2fetch(src_ptr, ne00 * sizeof(float), ne00 * sizeof(float), 1); + + if (k <= 64 && ne00 > 1024) { + float top_values[64]; + int32_t top_indices[64]; + top_k_select_tiled(src_ptr, ne00, k, values_buf, indices_buf, top_values, top_indices); + memcpy(dst_ptr, top_indices, k * sizeof(int32_t)); + continue; + } + + hvx_copy_f32_au((uint8_t*)values_buf, src_ptr, ne00); + + // Fills the indices ramp itself, so no init needed here. + if (ne00_padded > ne00) { + hvx_splat_f32_u((uint8_t *)(values_buf + ne00), -INFINITY, ne00_padded - ne00); + } + bitonic_sort_vtcm_desc((uint8_t*)values_buf, (uint8_t*)indices_buf, n_vec_pow2, true); + + // Copy top-k indices back to DDR + hvx_copy_f32_ua(dst_ptr, (const uint8_t *) indices_buf, k); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, start_row); +} + +// Single row (ne01=ne02=ne03=1) + large ne00 would otherwise run on one +// HVX thread while the rest sit idle. Split the row into n_chunks +// power-of-two chunks, sort each in parallel with bitonic_sort_vtcm_desc, +// then merge the n_chunks*local_k winners with one more sort that carries +// the global index through instead of re-deriving it. +struct htp_top_k_chunk_ctx { + struct htp_ops_context * octx; + uint8_t * vtcm_base; + size_t phase1_slot_size; + uint32_t ne00; + uint32_t chunk_elems; + uint32_t local_k; + size_t merge_values_off; + size_t merge_indices_off; +}; + +static void htp_top_k_chunk_job(unsigned int n, unsigned int i, void * data) { + struct htp_top_k_chunk_ctx * cctx = (struct htp_top_k_chunk_ctx *) data; + struct htp_ops_context * octx = cctx->octx; + const struct htp_tensor * src0 = octx->src[0]; + + uint32_t chunk_elems = cctx->chunk_elems; + uint32_t ne00 = cctx->ne00; + uint32_t local_k = cctx->local_k; + uint32_t chunk_base = i * chunk_elems; + + uint8_t * spad = cctx->vtcm_base + cctx->phase1_slot_size * i; + size_t values_size = hex_round_up(chunk_elems * sizeof(float), 128); + float * values_buf = (float *) spad; + int32_t * indices_buf = (int32_t *) (spad + values_size); + + uint32_t real_count = (chunk_base < ne00) ? MIN(chunk_elems, ne00 - chunk_base) : 0; + + if (real_count == 0) { + float * merge_values = (float *) (cctx->vtcm_base + cctx->merge_values_off); + int32_t * merge_indices = (int32_t *) (cctx->vtcm_base + cctx->merge_indices_off); + for (uint32_t j = 0; j < local_k; j++) { + merge_values[i * local_k + j] = -INFINITY; + merge_indices[i * local_k + j] = 0; + } + return; + } + + if (local_k > 1 && local_k <= 64 && chunk_elems > 1024) { + uint8_t * src_ptr = (uint8_t *) src0->data + (size_t) chunk_base * sizeof(float); + float top_values[64]; + int32_t top_indices[64]; + float * merge_values = (float *) (cctx->vtcm_base + cctx->merge_values_off); + int32_t * merge_indices = (int32_t *) (cctx->vtcm_base + cctx->merge_indices_off); + for (uint32_t j = 0; j < local_k; j++) { + top_values[j] = -INFINITY; + top_indices[j] = 0; + } + top_k_select_tiled(src_ptr, real_count, local_k, values_buf, indices_buf, top_values, top_indices); + + for (uint32_t j = 0; j < local_k; j++) { + merge_values[i * local_k + j] = top_values[j]; + merge_indices[i * local_k + j] = top_indices[j] + (int32_t) chunk_base; + } + return; + } + + if (real_count > 0) { + uint8_t * src_ptr = (uint8_t *) src0->data + (size_t) chunk_base * sizeof(float); + hex_l2fetch(src_ptr, real_count * sizeof(float), real_count * sizeof(float), 1); + hvx_copy_f32_au((uint8_t *) values_buf, src_ptr, real_count); + } + if (chunk_elems > real_count) { + hvx_splat_f32_u((uint8_t *) (values_buf + real_count), -INFINITY, chunk_elems - real_count); + } + + if (local_k == 1 && cctx->ne00 >= 128*1024) { + float max_value; + uint32_t max_index = top_k_max_value_index(values_buf, real_count, &max_value); + float * merge_values = (float *) (cctx->vtcm_base + cctx->merge_values_off); + int32_t * merge_indices = (int32_t *) (cctx->vtcm_base + cctx->merge_indices_off); + merge_values[i] = max_value; + merge_indices[i] = (int32_t) (max_index + chunk_base); + return; + } + + // chunk_elems is always a power-of-two multiple of 32 + bitonic_sort_vtcm_desc((uint8_t *) values_buf, (uint8_t *) indices_buf, chunk_elems / 32, true); + + float * merge_values = (float *) (cctx->vtcm_base + cctx->merge_values_off); + int32_t * merge_indices = (int32_t *) (cctx->vtcm_base + cctx->merge_indices_off); + + for (uint32_t j = 0; j < local_k; j++) { + merge_values[i * local_k + j] = values_buf[j]; + merge_indices[i * local_k + j] = indices_buf[j] + (int32_t) chunk_base; + } +} + +struct htp_top_k_merge_ctx { + struct htp_ops_context * octx; + uint8_t * vtcm_base; + size_t merge_values_off; + size_t merge_indices_off; + uint32_t merge_elems; + uint32_t total_candidates; + uint32_t k; +}; + +static void htp_top_k_merge_job(unsigned int n, unsigned int i, void * data) { + struct htp_top_k_merge_ctx * mctx = (struct htp_top_k_merge_ctx *) data; + struct htp_ops_context * octx = mctx->octx; + const struct htp_tensor * dst = octx->dst; + + float * merge_values = (float *) (mctx->vtcm_base + mctx->merge_values_off); + int32_t * merge_indices = (int32_t *) (mctx->vtcm_base + mctx->merge_indices_off); + + if (mctx->merge_elems > mctx->total_candidates) { + uint32_t pad = mctx->merge_elems - mctx->total_candidates; + hvx_splat_f32_u((uint8_t *) (merge_values + mctx->total_candidates), -INFINITY, pad); + for (uint32_t j = mctx->total_candidates; j < mctx->merge_elems; j++) { + merge_indices[j] = 0; + } + } + + // Preserve the global indices computed in phase 1 -- init_indices=false + // so they aren't overwritten with a local ramp. + bitonic_sort_vtcm_desc((uint8_t *) merge_values, (uint8_t *) merge_indices, mctx->merge_elems / 32, false); + + hvx_copy_f32_ua((uint8_t *) dst->data, (const uint8_t *) merge_indices, mctx->k); +} + +static int op_top_k_single_row_threaded(struct htp_ops_context * octx, uint32_t ne00, uint32_t k) { + uint32_t n_threads_avail = octx->n_threads; + + uint32_t n_vec = hmx_ceil_div(ne00, 32); + uint32_t n_vec_pow2 = 1; + while (n_vec_pow2 < n_vec) n_vec_pow2 <<= 1; + + // Largest power-of-two chunk count that both fits the available + // threads and evenly divides n_vec_pow2 + uint32_t n_chunks = 1; + while (n_chunks * 2 <= n_threads_avail && n_chunks * 2 <= n_vec_pow2) { + n_chunks *= 2; + } + + uint32_t chunk_n_vec = n_vec_pow2 / n_chunks; + uint32_t chunk_elems = chunk_n_vec * 32; + uint32_t local_k = MIN(k, chunk_elems); + + uint32_t total_candidates = n_chunks * local_k; + uint32_t merge_n_vec = hmx_ceil_div(total_candidates, 32); + uint32_t merge_n_vec_pow2 = 1; + while (merge_n_vec_pow2 < merge_n_vec) merge_n_vec_pow2 <<= 1; + uint32_t merge_elems = merge_n_vec_pow2 * 32; + + size_t phase1_values_size = hex_round_up(chunk_elems * sizeof(float), 128); + size_t phase1_indices_size = hex_round_up(chunk_elems * sizeof(int32_t), 128); + size_t phase1_slot_size = hex_round_up(phase1_values_size + phase1_indices_size, 256); + size_t phase1_total_size = phase1_slot_size * n_chunks; + + size_t merge_values_size = hex_round_up(merge_elems * sizeof(float), 128); + size_t merge_indices_size = hex_round_up(merge_elems * sizeof(int32_t), 128); + size_t merge_values_off = phase1_total_size; + size_t merge_indices_off = merge_values_off + merge_values_size; + + size_t total_vtcm = phase1_total_size + merge_values_size + merge_indices_size; + if (octx->ctx->vtcm_size < total_vtcm) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + + struct htp_top_k_chunk_ctx cctx; + cctx.octx = octx; + cctx.vtcm_base = vtcm_base; + cctx.phase1_slot_size = phase1_slot_size; + cctx.ne00 = ne00; + cctx.chunk_elems = chunk_elems; + cctx.local_k = local_k; + cctx.merge_values_off = merge_values_off; + cctx.merge_indices_off = merge_indices_off; + + work_queue_run(octx->ctx->work_queue, htp_top_k_chunk_job, &cctx, n_chunks); + + struct htp_top_k_merge_ctx mctx; + mctx.octx = octx; + mctx.vtcm_base = vtcm_base; + mctx.merge_values_off = merge_values_off; + mctx.merge_indices_off = merge_indices_off; + mctx.merge_elems = merge_elems; + mctx.total_candidates = total_candidates; + mctx.k = k; + + work_queue_run(octx->ctx->work_queue, htp_top_k_merge_job, &mctx, 1); + + return HTP_STATUS_OK; +} + +int op_top_k(struct htp_ops_context * octx) { + // Check supported types + if (octx->src[0]->type != HTP_TYPE_F32) { + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_row_size = dst->ne[0] * sizeof(int32_t); + + uint32_t row_start = 0; + uint32_t row_end = total_rows; + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(int32_t), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + row_end = range.start + range.count; + } + + const uint32_t nrows = row_end - row_start; + if (nrows == 0) { + return HTP_STATUS_OK; + } + + uint32_t ne00 = src0->ne[0]; + uint32_t k = dst->ne[0]; + + // Single row + large ne00: the per-row dispatch below would run on one + // HVX thread while the rest sit idle. Split the row across threads. + if (total_rows == 1 && ne00 > 1024) { + int status = op_top_k_single_row_threaded(octx, ne00, k); + if (status != HTP_STATUS_VTCM_TOO_SMALL) { + return status; + } + // else: fall through to the single-thread path below. + } + + const uint32_t n_threads = MIN(nrows, octx->n_threads); + + // Scratchpad layout: values + indices + // For bitonic: need padding to power-of-2 size + // Allocate for worst case (bitonic with padding) + uint32_t n_vec = hmx_ceil_div(ne00, 32); + uint32_t n_vec_pow2 = 1; + while (n_vec_pow2 < n_vec) n_vec_pow2 <<= 1; + uint32_t ne00_padded = n_vec_pow2 * 32; + + size_t values_size = hex_round_up(ne00_padded * sizeof(float), 128); + size_t indices_size = hex_round_up(ne00_padded * sizeof(int32_t), 128); + size_t spad_per_thread = values_size + indices_size; + + // Make sure we round up to 256 for alignment requirements + spad_per_thread = hex_round_up(spad_per_thread, 256); + + size_t total_spad_size = spad_per_thread * n_threads; + + if (octx->ctx->vtcm_size < total_spad_size) { + FARF(ERROR, "top_k: VTCM size too small. Needed %zu, have %zu", total_spad_size, octx->ctx->vtcm_size); + return HTP_STATUS_VTCM_TOO_SMALL; + } + + FARF(HIGH, "top_k: %ux%ux%ux%u -> %ux%ux%ux%u (0x%x, 0x%x)", + octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], + octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], + octx->src[0]->data, octx->dst->data); + + struct htp_top_k_context actx; + const struct fastdiv_values n_threads_div = init_fastdiv_values(n_threads); + actx.octx = octx; + actx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &n_threads_div); + actx.row_start = row_start; + actx.row_end = row_end; + actx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + actx.vtcm_per_thread = spad_per_thread; + actx.k = k; + + worker_callback_t job_func = htp_top_k_f32_fallback; + switch (ne00) { + case 1024: job_func = htp_top_k_f32_1024; break; + case 512: job_func = htp_top_k_f32_512; break; + case 256: job_func = htp_top_k_f32_256; break; + case 128: job_func = htp_top_k_f32_128; break; + case 64: job_func = htp_top_k_f32_64; break; + case 32: job_func = htp_top_k_f32_32; break; + default: job_func = htp_top_k_f32_fallback; break; + } + + // Run jobs + work_queue_run(octx->ctx->work_queue, job_func, &actx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c index db6177963541..155e852376f7 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.c +++ b/ggml/src/ggml-hexagon/htp/binary-ops.c @@ -8,14 +8,16 @@ #include <math.h> #include <string.h> -#include "hex-dma.h" +#include "dma-queue.h" #include "hvx-utils.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" +#include "binary-ops.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" #include "htp-tensor.h" #ifndef MIN @@ -25,6 +27,8 @@ // Context for binary operations struct htp_binary_context { struct htp_ops_context * octx; + struct htp_binary_vtcm_layout vtcm_layout; + uint8_t * vtcm_base; struct fastdiv_values src0_dim1_div; // ne01 struct fastdiv_values src0_dim2_div; // ne02 @@ -36,39 +40,44 @@ struct htp_binary_context { uint32_t block_max; uint32_t nrows_per_thread; + uint32_t total_rows; + uint32_t row_start; size_t src0_row_size_aligned; size_t src1_row_size_aligned; size_t dst_row_size_aligned; + size_t row_size_bytes; bool split_at_ne01; bool split_at_ne02; + + void * compute; }; #define htp_binary_preamble \ const struct htp_tensor * src0 = octx->src[0]; \ const struct htp_tensor * src1 = octx->src[1]; \ const struct htp_tensor * dst = octx->dst; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t ne10 = src1->ne[0]; \ - const uint32_t ne11 = src1->ne[1]; \ - const uint32_t ne12 = src1->ne[2]; \ - const uint32_t ne13 = src1->ne[3]; \ - \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb11 = src1->nb[1]; \ - const uint32_t nb12 = src1->nb[2]; \ - const uint32_t nb13 = src1->nb[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ + \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t ne10 = src1->ne[0]; \ + const uint32_t ne11 = src1->ne[1]; \ + const uint32_t ne12 = src1->ne[2]; \ + const uint32_t ne13 = src1->ne[3]; \ + \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb11 = src1->nb[1]; \ + const uint32_t nb12 = src1->nb[2]; \ + const uint32_t nb13 = src1->nb[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_t ir, uint32_t end_row, uint32_t ne01, uint32_t ne02) { @@ -92,115 +101,302 @@ static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_ return MIN(bctx->block_max, block_limit); } -// Macro for scalar op switch -#define COMPUTE_SCALAR_OP(DST, SRC, VAL, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ - case HTP_OP_SUB: hvx_sub_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ - case HTP_OP_MUL: hvx_mul_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ - case HTP_OP_DIV: hvx_mul_scalar_f32_aa(DST, SRC, 1.0f / (*(float *)VAL), N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_SUB: hvx_sub_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_MUL: hvx_mul_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_DIV: hvx_div_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - default: break; \ - } \ - } +// Out-of-line compute micro-kernels + +typedef void (*compute_scalar_dma_t)( + uint8_t * dst, const uint8_t * src0, const void * s1_table, + uint32_t cur_i11, uint32_t ne11, uint32_t n_rows, + size_t dst_stride, size_t src0_stride, uint32_t ne00); + +#define DEFINE_COMPUTE_SCALAR_DMA(NAME, TYPE, HVX_STMT) \ +static void compute_scalar_dma_##NAME( \ + uint8_t * dst, const uint8_t * src0, const void * s1_table, \ + uint32_t cur_i11, uint32_t ne11, uint32_t n_rows, \ + size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ + const TYPE * table = (const TYPE *) s1_table; \ + for (uint32_t r = 0; r < n_rows; r++) { \ + uint8_t * r_dst = dst + r * dst_stride; \ + const uint8_t * r_src0 = src0 + r * src0_stride; \ + TYPE val = table[cur_i11]; \ + HVX_STMT; \ + if (ne11 > 1 && ++cur_i11 == ne11) { \ + cur_i11 = 0; \ + } \ + } \ +} + +DEFINE_COMPUTE_SCALAR_DMA(add_f32, float, hvx_add_scalar_f32_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR_DMA(add_f16, _Float16, hvx_add_scalar_f16_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR_DMA(sub_f32, float, hvx_sub_scalar_f32_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR_DMA(sub_f16, _Float16, hvx_sub_scalar_f16_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR_DMA(mul_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR_DMA(mul_f16, _Float16, hvx_mul_scalar_f16_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR_DMA(div_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, 1.0f / (val), ne00)) +DEFINE_COMPUTE_SCALAR_DMA(div_f16, _Float16, hvx_div_scalar_f16_aa(r_dst, r_src0, val, ne00)) + +typedef void (*compute_scalar_t)( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_ptr, uint32_t s1_stride, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); + +#define DEFINE_COMPUTE_SCALAR(NAME, TYPE, HVX_STMT) \ +static void compute_scalar_##NAME( \ + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_ptr, uint32_t s1_stride, \ + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ + for (uint32_t r = 0; r < n_rows; r++) { \ + uint8_t * r_dst = dst + r * dst_stride; \ + const uint8_t * r_src0 = src0 + r * src0_stride; \ + TYPE val = *(const TYPE *)(src1_ptr + r * s1_stride); \ + HVX_STMT; \ + } \ +} + +DEFINE_COMPUTE_SCALAR(add_f32, float, hvx_add_scalar_f32_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(add_f16, _Float16, hvx_add_scalar_f16_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(sub_f32, float, hvx_sub_scalar_f32_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(sub_f16, _Float16, hvx_sub_scalar_f16_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(mul_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(mul_f16, _Float16, hvx_mul_scalar_f16_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(div_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, 1.0f / (val), ne00)) +DEFINE_COMPUTE_SCALAR(div_f16, _Float16, hvx_div_scalar_f16_aa(r_dst, r_src0, val, ne00)) + +typedef void (*compute_same_shape_t)( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, size_t src1_stride, uint32_t ne00); + +#define DEFINE_COMPUTE_SAME_SHAPE(NAME, HVX_FN) \ +static void compute_same_shape_##NAME( \ + uint8_t * dst, const uint8_t * src0, const uint8_t * src1, \ + uint32_t n_rows, size_t dst_stride, size_t src0_stride, size_t src1_stride, uint32_t ne00) { \ + for (uint32_t r = 0; r < n_rows; r++) { \ + HVX_FN(dst + r * dst_stride, src0 + r * src0_stride, src1 + r * src1_stride, ne00); \ + } \ +} + +DEFINE_COMPUTE_SAME_SHAPE(add_f32, hvx_add_f32_aaa) +DEFINE_COMPUTE_SAME_SHAPE(add_f16, hvx_add_f16_aaa) +DEFINE_COMPUTE_SAME_SHAPE(sub_f32, hvx_sub_f32_aaa) +DEFINE_COMPUTE_SAME_SHAPE(sub_f16, hvx_sub_f16_aaa) +DEFINE_COMPUTE_SAME_SHAPE(mul_f32, hvx_mul_f32_aaa) +DEFINE_COMPUTE_SAME_SHAPE(mul_f16, hvx_mul_f16_aaa) +DEFINE_COMPUTE_SAME_SHAPE(div_f32, hvx_div_f32_aaa) +DEFINE_COMPUTE_SAME_SHAPE(div_f16, hvx_div_f16_aaa) + +typedef void (*compute_row_bcast_t)( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); + +#define DEFINE_COMPUTE_ROW_BCAST(NAME, HVX_FN) \ +static void compute_row_bcast_##NAME( \ + uint8_t * dst, const uint8_t * src0, const uint8_t * src1, \ + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ + for (uint32_t r = 0; r < n_rows; r++) { \ + HVX_FN(dst + r * dst_stride, src0 + r * src0_stride, src1, ne00); \ + } \ +} + +DEFINE_COMPUTE_ROW_BCAST(add_f32, hvx_add_f32_aaa) +DEFINE_COMPUTE_ROW_BCAST(add_f16, hvx_add_f16_aaa) +DEFINE_COMPUTE_ROW_BCAST(sub_f32, hvx_sub_f32_aaa) +DEFINE_COMPUTE_ROW_BCAST(sub_f16, hvx_sub_f16_aaa) +DEFINE_COMPUTE_ROW_BCAST(mul_f32, hvx_mul_f32_aaa) +DEFINE_COMPUTE_ROW_BCAST(mul_f16, hvx_mul_f16_aaa) +DEFINE_COMPUTE_ROW_BCAST(div_f32, hvx_div_f32_aaa) +DEFINE_COMPUTE_ROW_BCAST(div_f16, hvx_div_f16_aaa) + +typedef void (*compute_complex_t)( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); + +#define DEFINE_COMPUTE_COMPLEX(NAME, HVX_FN) \ +static void compute_complex_##NAME( \ + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \ + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, \ + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ + for (uint32_t r = 0; r < n_rows; r++) { \ + uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \ + const uint8_t * r_src1 = src1_plane + i11 * nb11; \ + HVX_FN(dst + r * dst_stride, src0 + r * src0_stride, r_src1, ne00); \ + } \ +} + +DEFINE_COMPUTE_COMPLEX(add_f32, hvx_add_f32_aau) +DEFINE_COMPUTE_COMPLEX(add_f16, hvx_add_f16_aau) +DEFINE_COMPUTE_COMPLEX(sub_f32, hvx_sub_f32_aau) +DEFINE_COMPUTE_COMPLEX(sub_f16, hvx_sub_f16_aau) +DEFINE_COMPUTE_COMPLEX(mul_f32, hvx_mul_f32_aau) +DEFINE_COMPUTE_COMPLEX(mul_f16, hvx_mul_f16_aau) +DEFINE_COMPUTE_COMPLEX(div_f32, hvx_div_f32_aau) +DEFINE_COMPUTE_COMPLEX(div_f16, hvx_div_f16_aau) + +typedef void (*compute_repeat_t)( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10); + +#define DEFINE_COMPUTE_REPEAT(NAME, TYPE, HVX_FN) \ +static void compute_repeat_##NAME( \ + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \ + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, \ + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10) { \ + for (uint32_t r = 0; r < n_rows; r++) { \ + uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \ + const uint8_t * r_src1_row = src1_plane + i11 * nb11; \ + uint8_t * r_dst = dst + r * dst_stride; \ + const uint8_t * r_src0 = src0 + r * src0_stride; \ + for (uint32_t c = 0; c < ne00; c += ne10) { \ + uint32_t len = MIN(ne10, ne00 - c); \ + HVX_FN(r_dst + c * sizeof(TYPE), r_src0 + c * sizeof(TYPE), r_src1_row, len); \ + } \ + } \ +} -// Macro for vector op switch (All Aligned) -#define COMPUTE_VECTOR_OP_AAA(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f32_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f32_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f32_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f32_aaa(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f16_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f16_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f16_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f16_aaa(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ +DEFINE_COMPUTE_REPEAT(add_f32, float, hvx_add_f32_uuu) +DEFINE_COMPUTE_REPEAT(add_f16, _Float16, hvx_add_f16_uuu) +DEFINE_COMPUTE_REPEAT(sub_f32, float, hvx_sub_f32_uuu) +DEFINE_COMPUTE_REPEAT(sub_f16, _Float16, hvx_sub_f16_uuu) +DEFINE_COMPUTE_REPEAT(mul_f32, float, hvx_mul_f32_uuu) +DEFINE_COMPUTE_REPEAT(mul_f16, _Float16, hvx_mul_f16_uuu) +DEFINE_COMPUTE_REPEAT(div_f32, float, hvx_div_f32_uuu) +DEFINE_COMPUTE_REPEAT(div_f16, _Float16, hvx_div_f16_uuu) + +typedef void (*compute_add_id_t)( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const char * src2_data, + uint32_t i01, uint32_t i02, uint32_t nb20, uint32_t nb21, uint32_t src1_stride, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); + +static void compute_add_id_f32( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const char * src2_data, + uint32_t i01, uint32_t i02, uint32_t nb20, uint32_t nb21, uint32_t src1_stride, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { + for (uint32_t r = 0; r < n_rows; r++) { + uint32_t r_i01 = i01 + r; + const int32_t idx = *(const int32_t *)(src2_data + r_i01 * nb20 + i02 * nb21); + if (idx < 0) { + memcpy(dst + r * dst_stride, src0 + r * src0_stride, ne00 * sizeof(float)); + continue; + } + const uint8_t * r_src1 = src1_data + idx * src1_stride; + const uint8_t * r_src0 = src0 + r * src0_stride; + uint8_t * r_dst = dst + r * dst_stride; + hvx_add_f32_aaa(r_dst, r_src0, r_src1, ne00); } +} + +// 1a. Scalar src1 in VTCM via DMA (ne10 == 1, ne12 == 1, ne13 == 1) +static void binary_thread_scalar_dma(unsigned int nth, unsigned int ith, void * data) { + struct htp_binary_context * bctx = (struct htp_binary_context *) data; + struct htp_ops_context * octx = bctx->octx; + htp_binary_preamble; + + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + if (start_row >= end_row) return; + + FARF(HIGH, "binary-scalar-dma: %d/%d (%u:%u) row-size %u (%u)", + ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + size_t src0_spad_half = layout->src0_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; + const void * s1_table = VTCM_LAYOUT_PTR(const void, bctx->vtcm_base, layout->off_src1); + + dma_queue * dma_q = octx->ctx->dma[ith]; + uint32_t ir_prefetch = start_row; + int spad_idx = 0; + + for (int k = 0; k < 2 && ir_prefetch < end_row; k++) { + uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t i03, i02, i01, rem; + i03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + rem = ir_prefetch - i03 * (ne02 * ne01); + i02 = fastdiv(rem, &bctx->src0_dim1_div); + i01 = rem - i02 * ne01; + + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; -// Macro for vector op switch (Dst Aligned, Src0 Aligned, Src1 Unaligned) -#define COMPUTE_VECTOR_OP_AAU(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f32_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f32_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f32_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f32_aau(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f16_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f16_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f16_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f16_aau(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ + uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; + uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; + + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, + current_block_size); + ir_prefetch += current_block_size; + spad_idx ^= 1; } -// Macro for vector op switch (All Unaligned - generic loop used in element repeat) -#define COMPUTE_VECTOR_OP_UUU(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f32_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f32_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f32_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f32_uuu(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f16_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f16_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f16_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f16_uuu(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + compute_scalar_dma_t compute = (compute_scalar_dma_t) bctx->compute; + + for (uint32_t ir = start_row; ir < end_row; ) { + uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); + + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + + uint32_t i03, i02, i01, rem; + i03 = fastdiv(ir, &bctx->src0_dim12_div); + rem = ir - i03 * (ne02 * ne01); + i02 = fastdiv(rem, &bctx->src0_dim1_div); + i01 = rem - i02 * ne01; + + uint32_t cur_i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute(d_spad, s0_spad, s1_table, cur_i11, ne11, current_block_size, + bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, + current_block_size); + + if (ir_prefetch < end_row) { + uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, + next_block_size); + ir_prefetch += next_block_size; + } + ir += current_block_size; } -// 1. Scalar src1 (ne10 == 1) -static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { + dma_queue_flush(dma_q); +} + +// 1b. Scalar src1 dynamic / pointer (ne10 == 1) +static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; - const uint32_t src0_type = octx->src[0]->type; - const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", + ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + size_t src0_spad_half = layout->src0_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; int spad_idx = 0; - // Preamble for (int k = 0; k < 2 && ir_prefetch < end_row; k++) { uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); uint32_t i03, i02, i01, rem; @@ -209,24 +405,26 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { i02 = fastdiv(rem, &bctx->src0_dim1_div); i01 = rem - i02 * ne01; - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); ir_prefetch += current_block_size; spad_idx ^= 1; } - // Main loop + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + compute_scalar_t compute = (compute_scalar_t) bctx->compute; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i03, i02, i01, rem; i03 = fastdiv(ir, &bctx->src0_dim12_div); @@ -234,65 +432,62 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { i02 = fastdiv(rem, &bctx->src0_dim1_div); i01 = rem - i02 * ne01; - // src1 indices (broadcast/repeat) uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); uint32_t i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div); - uint8_t * src1_ptr = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; + const uint8_t * src1_ptr = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; uint32_t s1_stride = (ne11 == 1) ? 0 : nb11; - for (uint32_t r = 0; r < current_block_size; r++) { - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; - COMPUTE_SCALAR_OP(r_dst, r_src0, src1_ptr, src0_type, ne00); - src1_ptr += s1_stride; - } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute(d_spad, s0_spad, src1_ptr, s1_stride, current_block_size, + bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03, p02, p01, prem; - p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - prem = ir_prefetch - p03 * (ne02 * ne01); - p02 = fastdiv(prem, &bctx->src0_dim1_div); - p01 = prem - p02 * ne01; - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); - ir_prefetch += next_block_size; + uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); + ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); } // 2. Vector Same Shape (ne1x == ne0x) or Simple Broadcast -static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; - const uint32_t src0_type = octx->src[0]->type; - const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - FARF(HIGH, "binary-same-shape: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + FARF(HIGH, "binary-same-shape: %d/%d (%u:%u) row-size %u (%u)", + ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * src1_spad_base = octx->src1_spad.data + (ith * octx->src1_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * src1_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t src1_spad_half = octx->src1_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + size_t src0_spad_half = layout->src0_spad_half_size; + size_t src1_spad_half = layout->src1_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -308,94 +503,95 @@ static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, voi uint32_t i12 = (ne12 == 1) ? 0 : i02; uint32_t i11 = (ne11 == 1) ? 0 : i01; - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * src1_curr = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t src1_curr = src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * s1_spad = src1_spad_base + spad_idx * src1_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); - dma_queue_push(q, dma_make_ptr(s1_spad, src1_curr), bctx->src1_row_size_aligned, nb11, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(s1_spad, src1_curr), bctx->src1_row_size_aligned, nb11, row_size_bytes, current_block_size); ir_prefetch += current_block_size; spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + compute_same_shape_t compute = (compute_same_shape_t) bctx->compute; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; - uint8_t * s1_spad = (uint8_t *) dma_queue_pop(q).dst; - - for (uint32_t r = 0; r < current_block_size; r++) { - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_src1 = s1_spad + r * bctx->src1_row_size_aligned; - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; - COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00); - } + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + uint8_t * s1_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute(d_spad, s0_spad, s1_spad, current_block_size, + bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, bctx->src1_row_size_aligned, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint32_t i03, i02, i01, rem; i03 = fastdiv(ir, &bctx->src0_dim12_div); rem = ir - i03 * (ne02 * ne01); i02 = fastdiv(rem, &bctx->src0_dim1_div); i01 = rem - i02 * ne01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03, p02, p01, prem; - p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - prem = ir_prefetch - p03 * (ne02 * ne01); - p02 = fastdiv(prem, &bctx->src0_dim1_div); - p01 = prem - p02 * ne01; + uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; - uint32_t p13 = (ne13 == 1) ? 0 : p03; - uint32_t p12 = (ne12 == 1) ? 0 : p02; - uint32_t p11 = (ne11 == 1) ? 0 : p01; + uint32_t p13 = (ne13 == 1) ? 0 : p03; + uint32_t p12 = (ne12 == 1) ? 0 : p02; + uint32_t p11 = (ne11 == 1) ? 0 : p01; - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - uint8_t * s1_next = (uint8_t *)src1->data + p13 * nb13 + p12 * nb12 + p11 * nb11; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_addr_t s1_next = src1->data + p13 * nb13 + p12 * nb12 + p11 * nb11; - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); - dma_queue_push(q, dma_make_ptr(s1_spad, s1_next), bctx->src1_row_size_aligned, nb11, row_size_bytes, next_block_size); + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); + dma_queue_push(dma_q, dma_make_data(s1_spad, s1_next), bctx->src1_row_size_aligned, nb11, row_size_bytes, next_block_size); - ir_prefetch += next_block_size; + ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); } // 3. Row Broadcast (ne11 == 1, ne12 == 1, single row src1) -static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; - const uint32_t src0_type = octx->src[0]->type; - const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - FARF(HIGH, "binary-row-bcast: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + FARF(HIGH, "binary-row-bcast: %d/%d (%u:%u) row-size %u (%u)", + ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * src1_spad_base = octx->src1_spad.data + (ith * octx->src1_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + size_t src0_spad_half = layout->src0_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; int spad_idx = 0; - void * s1_ptr = (void *) src1_spad_base; + void * s1_ptr = VTCM_LAYOUT_PTR(void, bctx->vtcm_base, layout->off_src1); for (int k = 0; k < 2 && ir_prefetch < end_row; k++) { uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); @@ -404,73 +600,76 @@ static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith, uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); ir_prefetch += current_block_size; spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + compute_row_bcast_t compute = (compute_row_bcast_t) bctx->compute; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; - - for (uint32_t r = 0; r < current_block_size; r++) { - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_src1 = (uint8_t *)s1_ptr; // Constant - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; - COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00); - } + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute(d_spad, s0_spad, (const uint8_t *)s1_ptr, current_block_size, + bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint32_t i03 = fastdiv(ir, &bctx->src0_dim12_div); uint32_t rem = ir - i03 * (ne02 * ne01); uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - uint32_t prem = ir_prefetch - p03 * (ne02 * ne01); - uint32_t p02 = fastdiv(prem, &bctx->src0_dim1_div); - uint32_t p01 = prem - p02 * ne01; - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); - ir_prefetch += next_block_size; + uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); + ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); } // 4. Vector Complex (ne10 == ne00, complex broadcast) -static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; - const uint32_t src0_type = octx->src[0]->type; - const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - FARF(HIGH, "binary-complex: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + FARF(HIGH, "binary-complex: %d/%d (%u:%u) row-size %u (%u)", + ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + size_t src0_spad_half = layout->src0_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -481,82 +680,81 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); ir_prefetch += current_block_size; spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + compute_complex_t compute = (compute_complex_t) bctx->compute; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i03 = fastdiv(ir, &bctx->src0_dim12_div); uint32_t rem = ir - i03 * (ne02 * ne01); uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - for (uint32_t r = 0; r < current_block_size; r++) { - uint32_t r_i01 = i01 + r; - uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); - uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); - uint32_t i11 = fastmodulo(r_i01, ne11, &bctx->src1_dim1_div); + uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); + uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); + const uint8_t * src1_plane = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12; - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_src1 = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, nb11, + current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - // Read src1 from DDR (unaligned) - COMPUTE_VECTOR_OP_AAU(r_dst, r_src0, r_src1, src0_type, ne00); - } - - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - uint32_t prem = ir_prefetch - p03 * (ne02 * ne01); - uint32_t p02 = fastdiv(prem, &bctx->src0_dim1_div); - uint32_t p01 = prem - p02 * ne01; - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); - ir_prefetch += next_block_size; + uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); + ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); } // 5. Element Repeat (ne10 != ne00) -static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; - const uint32_t src0_type = octx->src[0]->type; - const uint32_t elem_size_bytes = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); - const uint32_t row_size_bytes = ne00 * elem_size_bytes;; - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + size_t src0_spad_half = layout->src0_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; - FARF(HIGH, "binary-repeat: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + FARF(HIGH, "binary-repeat: %d/%d (%u:%u) row-size %u (%u)", + ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -567,66 +765,62 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); ir_prefetch += current_block_size; spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + compute_repeat_t compute = (compute_repeat_t) bctx->compute; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i03 = fastdiv(ir, &bctx->src0_dim12_div); uint32_t rem = ir - i03 * (ne02 * ne01); uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - for (uint32_t r = 0; r < current_block_size; r++) { - uint32_t r_i01 = i01 + r; - uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); - uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); - uint32_t i11 = fastmodulo(r_i01, ne11, &bctx->src1_dim1_div); - - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_src1_row = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; - - // Repeat src1 row - for (uint32_t c = 0; c < ne00; c += ne10) { - uint32_t len = MIN(ne10, ne00 - c); - // Use UUU for speed and simplicity - COMPUTE_VECTOR_OP_UUU(r_dst + c * elem_size_bytes, r_src0 + c * elem_size_bytes, r_src1_row, src0_type, len); - } - } + uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); + uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); + const uint8_t * src1_plane = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, nb11, + current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00, ne10); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - uint32_t prem = ir_prefetch - p03 * (ne02 * ne01); - uint32_t p02 = fastdiv(prem, &bctx->src0_dim1_div); - uint32_t p01 = prem - p02 * ne01; - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); - ir_prefetch += next_block_size; + uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); + ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); } // 6. ADD_ID (src1 gathered via src2 indices) -static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; @@ -639,28 +833,29 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { const uint32_t ne01 = src0->ne[1]; const uint32_t ne02 = src0->ne[2]; const uint32_t ne03 = src0->ne[3]; - const uint32_t ne11 = src1->ne[1]; // for bounds check const uint32_t nb01 = src0->nb[1]; const uint32_t nb02 = src0->nb[2]; const uint32_t nb03 = src0->nb[3]; - const uint32_t nb11 = src1->nb[1]; // src1 row stride + const uint32_t src1_stride = bctx->src1_row_size_aligned; const uint32_t nb1 = dst->nb[1]; const uint32_t nb2 = dst->nb[2]; const uint32_t nb3 = dst->nb[3]; - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); + size_t src0_spad_half = layout->src0_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -671,155 +866,139 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, ne00 * sizeof(float), 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, ne00 * sizeof(float), current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); ir_prefetch += current_block_size; spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i03 = fastdiv(ir, &bctx->src0_dim12_div); uint32_t rem = ir - i03 * (ne02 * ne01); uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - for (uint32_t r = 0; r < current_block_size; r++) { - uint32_t r_i01 = i01 + r; // linear within block since we split at ne01 - - const int32_t idx = *(int32_t *)((char *)src2->data + r_i01 * src2->nb[0] + i02 * src2->nb[1]); - - uint8_t * r_src1 = (uint8_t *)src1->data + idx * nb11; - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute_add_id_t compute = (compute_add_id_t) bctx->compute; + compute(d_spad, s0_spad, vtcm_src1, (const char *)(uintptr_t)src2->data, + i01, i02, src2->nb[0], src2->nb[1], src1_stride, + current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - hvx_add_f32_aau(r_dst, r_src0, r_src1, ne00); - } - - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, ne00 * sizeof(float), current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); if (ir_prefetch < end_row) { uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - uint32_t prem = ir_prefetch - p03 * (ne02 * ne01); - uint32_t p02 = fastdiv(prem, &bctx->src0_dim1_div); - uint32_t p01 = prem - p02 * ne01; - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, ne00 * sizeof(float), next_block_size); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); } static int execute_op_binary(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; + const struct htp_binary_kernel_params * kparams = (const struct htp_binary_kernel_params *) octx->kernel_params; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - // Use packed row sizes for VTCM allocation + // Use packed row sizes for VTCM allocation and alignment const uint32_t src0_type = octx->src[0]->type; const size_t elem_size = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); const size_t src0_row_size = src0->ne[0] * elem_size; const size_t src1_row_size = src1->ne[0] * elem_size; const size_t dst_row_size = dst->ne[0] * elem_size; - size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); - size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN); - size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); - - bool is_add_id = (octx->op == HTP_OP_ADD_ID); - bool is_scalar = !is_add_id && (src1->ne[0] == 1); - - bool is_transposed = (src0->nb[1] < src0_row_size || src1->nb[1] < src1_row_size || dst->nb[1] < dst_row_size); - - bool is_same_shape = !is_add_id && !is_scalar && !is_transposed && - (src1->ne[0] == src0->ne[0] && src0->ne[0] % VLEN == 0) && - (src1->ne[1] == src0->ne[1] || src1->ne[1] == 1) && - (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) && - (src1->ne[3] == src0->ne[3] || src1->ne[3] == 1); - - bool is_row_bcast = is_same_shape && (src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1); - bool is_complex = !is_add_id && !is_scalar && !is_same_shape && (src1->ne[0] == src0->ne[0]); - bool is_repeat = !is_add_id && !is_scalar && !is_same_shape && (src1->ne[0] != src0->ne[0]); + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; - size_t spad_row_total; - if (is_same_shape) { - spad_row_total = 2 * (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned); - } else { - spad_row_total = 2 * (src0_row_size_aligned + dst_row_size_aligned); + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; } - size_t rows_per_buffer = octx->ctx->vtcm_size / (n_threads * spad_row_total); - - // Adjust for static src1 in row_bcast case - if (is_row_bcast) { - size_t needed_static = src1_row_size_aligned; - if (octx->ctx->vtcm_size < needed_static) return HTP_STATUS_VTCM_TOO_SMALL; - size_t avail = octx->ctx->vtcm_size - needed_static; - rows_per_buffer = avail / (n_threads * spad_row_total); + if (nrows == 0) { + return HTP_STATUS_OK; } - if (rows_per_buffer < 1) { - FARF(ERROR, "binary: VTCM too small\n"); - return HTP_STATUS_VTCM_TOO_SMALL; + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; } - octx->src0_spad.size_per_thread = rows_per_buffer * 2 * src0_row_size_aligned; - octx->dst_spad.size_per_thread = rows_per_buffer * 2 * dst_row_size_aligned; - - if (is_add_id || is_scalar || is_complex || is_repeat || is_row_bcast) { - octx->src1_spad.size_per_thread = 0; - } else { - octx->src1_spad.size_per_thread = rows_per_buffer * 2 * src1_row_size_aligned; - } - - octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread; - octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread; - if (is_row_bcast) { - octx->src1_spad.size = src1_row_size_aligned; - } else { - octx->src1_spad.size = n_threads * octx->src1_spad.size_per_thread; + const uint32_t n_threads = octx->n_threads; + const size_t src0_row_size_aligned = kparams->src0_row_size_aligned; + const size_t src1_row_size_aligned = kparams->src1_row_size_aligned; + const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; + + if (htp_tensor_is_extended(src1)) { + if (kparams->kernel_type != HTP_BINARY_KERNEL_SAME_SHAPE && + kparams->kernel_type != HTP_BINARY_KERNEL_ROW_BCAST && + kparams->kernel_type != HTP_BINARY_KERNEL_SCALAR_DMA && + kparams->kernel_type != HTP_BINARY_KERNEL_ADD_ID) { + return HTP_STATUS_NO_SUPPORT; + } } - if (octx->ctx->vtcm_size < (octx->src0_spad.size + octx->src1_spad.size + octx->dst_spad.size)) { - return HTP_STATUS_VTCM_TOO_SMALL; + if (octx->op == HTP_OP_ADD_ID && htp_tensor_is_extended(octx->src[2])) { + return HTP_STATUS_NO_SUPPORT; } - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->src1_spad.src = NULL; - octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; octx->dst_spad.src = NULL; + struct htp_binary_context bctx; + bctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + htp_binary_vtcm_layout_build(&bctx.vtcm_layout, kparams, octx->ctx->vtcm_size); - if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - return HTP_STATUS_OK; + if (bctx.vtcm_layout.rows_per_buffer == 0 || bctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; } - dma_queue * q = octx->ctx->dma[0]; - if (is_row_bcast) { - dma_queue_push(q, dma_make_ptr(octx->src1_spad.data, (const void *) src1->data), src1_row_size_aligned, 0, src1->ne[0] * elem_size, 1); + dma_queue * dma_q = octx->ctx->dma[0]; + uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src1); + if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) { + dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[0] * elem_size, 1); + } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR_DMA) { + dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[1] * elem_size, 1); + } else if (kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) { + dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), + kparams->src1_row_size_aligned, src1->nb[1], + src1->ne[0] * elem_size, src1->ne[1]); } - struct htp_binary_context bctx; bctx.octx = octx; - bctx.nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - bctx.block_max = rows_per_buffer; + bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + bctx.total_rows = nrows; + bctx.row_start = row_start; + bctx.block_max = bctx.vtcm_layout.rows_per_buffer; bctx.src0_row_size_aligned = src0_row_size_aligned; bctx.src1_row_size_aligned = src1_row_size_aligned; bctx.dst_row_size_aligned = dst_row_size_aligned; + bctx.row_size_bytes = src0_row_size; bctx.src0_dim1_div = init_fastdiv_values(src0->ne[1]); bctx.src0_dim2_div = init_fastdiv_values(src0->ne[2]); @@ -835,22 +1014,156 @@ static int execute_op_binary(struct htp_ops_context * octx) { bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]); bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]); - bctx.split_at_ne01 = (src0->ne[2] > 1) && ((src1->ne[1] > 1) || (src1->ne[2] > 1) || !src0_contig_dim1 || !dst_contig_dim1); + bctx.split_at_ne01 = (octx->op == HTP_OP_ADD_ID) || + ((src0->ne[2] > 1) && ((src1->ne[1] > 1) || (src1->ne[2] > 1) || !src0_contig_dim1 || !dst_contig_dim1)); bctx.split_at_ne02 = (src0->ne[3] > 1) && ((src1->ne[2] > 1) || (src1->ne[3] > 1) || !src0_contig_dim2 || !dst_contig_dim2); - worker_callback_t worker_func; - if (is_add_id) worker_func = binary_job_add_id; - else if (is_scalar) worker_func = binary_job_scalar; - else if (is_row_bcast) worker_func = binary_job_vector_row_broadcast; - else if (is_same_shape) worker_func = binary_job_vector_same_shape; - else if (is_complex) worker_func = binary_job_vector_complex; - else worker_func = binary_job_element_repeat; + worker_callback_t worker_func = NULL; + void * compute_func = NULL; + + switch (kparams->kernel_type) { + case HTP_BINARY_KERNEL_SAME_SHAPE: + worker_func = binary_thread_vector_same_shape; + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_same_shape_add_f32; break; + case HTP_OP_SUB: compute_func = compute_same_shape_sub_f32; break; + case HTP_OP_MUL: compute_func = compute_same_shape_mul_f32; break; + case HTP_OP_DIV: compute_func = compute_same_shape_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_same_shape_add_f16; break; + case HTP_OP_SUB: compute_func = compute_same_shape_sub_f16; break; + case HTP_OP_MUL: compute_func = compute_same_shape_mul_f16; break; + case HTP_OP_DIV: compute_func = compute_same_shape_div_f16; break; + default: break; + } + } + break; + case HTP_BINARY_KERNEL_ROW_BCAST: + worker_func = binary_thread_vector_row_broadcast; + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_row_bcast_add_f32; break; + case HTP_OP_SUB: compute_func = compute_row_bcast_sub_f32; break; + case HTP_OP_MUL: compute_func = compute_row_bcast_mul_f32; break; + case HTP_OP_DIV: compute_func = compute_row_bcast_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_row_bcast_add_f16; break; + case HTP_OP_SUB: compute_func = compute_row_bcast_sub_f16; break; + case HTP_OP_MUL: compute_func = compute_row_bcast_mul_f16; break; + case HTP_OP_DIV: compute_func = compute_row_bcast_div_f16; break; + default: break; + } + } + break; + case HTP_BINARY_KERNEL_SCALAR_DMA: + worker_func = binary_thread_scalar_dma; + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_scalar_dma_add_f32; break; + case HTP_OP_SUB: compute_func = compute_scalar_dma_sub_f32; break; + case HTP_OP_MUL: compute_func = compute_scalar_dma_mul_f32; break; + case HTP_OP_DIV: compute_func = compute_scalar_dma_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_scalar_dma_add_f16; break; + case HTP_OP_SUB: compute_func = compute_scalar_dma_sub_f16; break; + case HTP_OP_MUL: compute_func = compute_scalar_dma_mul_f16; break; + case HTP_OP_DIV: compute_func = compute_scalar_dma_div_f16; break; + default: break; + } + } + break; + case HTP_BINARY_KERNEL_SCALAR: + worker_func = binary_thread_scalar; + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_scalar_add_f32; break; + case HTP_OP_SUB: compute_func = compute_scalar_sub_f32; break; + case HTP_OP_MUL: compute_func = compute_scalar_mul_f32; break; + case HTP_OP_DIV: compute_func = compute_scalar_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_scalar_add_f16; break; + case HTP_OP_SUB: compute_func = compute_scalar_sub_f16; break; + case HTP_OP_MUL: compute_func = compute_scalar_mul_f16; break; + case HTP_OP_DIV: compute_func = compute_scalar_div_f16; break; + default: break; + } + } + break; + case HTP_BINARY_KERNEL_COMPLEX: + worker_func = binary_thread_vector_complex; + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_complex_add_f32; break; + case HTP_OP_SUB: compute_func = compute_complex_sub_f32; break; + case HTP_OP_MUL: compute_func = compute_complex_mul_f32; break; + case HTP_OP_DIV: compute_func = compute_complex_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_complex_add_f16; break; + case HTP_OP_SUB: compute_func = compute_complex_sub_f16; break; + case HTP_OP_MUL: compute_func = compute_complex_mul_f16; break; + case HTP_OP_DIV: compute_func = compute_complex_div_f16; break; + default: break; + } + } + break; + case HTP_BINARY_KERNEL_REPEAT: + worker_func = binary_thread_element_repeat; + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_repeat_add_f32; break; + case HTP_OP_SUB: compute_func = compute_repeat_sub_f32; break; + case HTP_OP_MUL: compute_func = compute_repeat_mul_f32; break; + case HTP_OP_DIV: compute_func = compute_repeat_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_repeat_add_f16; break; + case HTP_OP_SUB: compute_func = compute_repeat_sub_f16; break; + case HTP_OP_MUL: compute_func = compute_repeat_mul_f16; break; + case HTP_OP_DIV: compute_func = compute_repeat_div_f16; break; + default: break; + } + } + break; + case HTP_BINARY_KERNEL_ADD_ID: + if (octx->op == HTP_OP_ADD_ID && src0_type == HTP_TYPE_F32) { + worker_func = binary_thread_add_id_f32; + compute_func = (void *) compute_add_id_f32; + } + break; + default: break; + } + + if (!worker_func || !compute_func) { + return HTP_STATUS_NO_SUPPORT; + } - if (is_row_bcast) { - dma_queue_pop(q); + bctx.compute = compute_func; + + if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST || + kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR_DMA || + kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) { + dma_queue_pop(dma_q); } - worker_pool_run_func(octx->ctx->worker_pool, worker_func, &bctx, n_threads); + work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads); return HTP_STATUS_OK; } @@ -870,4 +1183,3 @@ int op_binary(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.h b/ggml/src/ggml-hexagon/htp/binary-ops.h new file mode 100644 index 000000000000..b99f2ad64f9b --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/binary-ops.h @@ -0,0 +1,111 @@ +#ifndef HTP_BINARY_OPS_H +#define HTP_BINARY_OPS_H + +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include "hex-common.h" +#include "htp-ops.h" +#include "htp-vtcm.h" + +enum htp_binary_kernel_type { + HTP_BINARY_KERNEL_SAME_SHAPE = 0, + HTP_BINARY_KERNEL_ROW_BCAST, + HTP_BINARY_KERNEL_SCALAR_DMA, + HTP_BINARY_KERNEL_SCALAR, + HTP_BINARY_KERNEL_ADD_ID, + HTP_BINARY_KERNEL_COMPLEX, + HTP_BINARY_KERNEL_REPEAT, +}; + +struct htp_binary_kernel_params { + uint32_t kernel_type; + uint32_t n_threads; + uint32_t rows_per_buffer; + + uint32_t src0_row_size_aligned; + uint32_t src1_row_size_aligned; + uint32_t dst_row_size_aligned; + + uint32_t src1_size; + uint32_t vtcm_size; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_binary_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_src1; + size_t off_dst; + + size_t src0_bytes_per_thread; + size_t src1_bytes_per_thread; + size_t dst_bytes_per_thread; + + size_t src0_spad_half_size; + size_t src1_spad_half_size; + size_t dst_spad_half_size; + + size_t src1_size; + uint32_t rows_per_buffer; +}; + +static inline void htp_binary_vtcm_layout_build( + struct htp_binary_vtcm_layout * L, + const struct htp_binary_kernel_params * kparams, + size_t vtcm_size +) { + memset(L, 0, sizeof(*L)); + + const uint32_t n_threads = kparams->n_threads; + if (n_threads == 0) { + return; + } + + const size_t spad_row_total = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) + ? 2 * (kparams->src0_row_size_aligned + kparams->src1_row_size_aligned + kparams->dst_row_size_aligned) + : 2 * (kparams->src0_row_size_aligned + kparams->dst_row_size_aligned); + + if (spad_row_total == 0 || vtcm_size < kparams->src1_size) { + return; + } + + const size_t rows_per_buffer = (vtcm_size - kparams->src1_size) / (n_threads * spad_row_total); + if (rows_per_buffer == 0) { + return; + } + + L->rows_per_buffer = (uint32_t) rows_per_buffer; + L->src1_size = kparams->src1_size; + + L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned; + L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned; + L->src1_bytes_per_thread = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) + ? rows_per_buffer * 2 * kparams->src1_row_size_aligned + : 0; + + L->src0_spad_half_size = L->src0_bytes_per_thread / 2; + L->src1_spad_half_size = L->src1_bytes_per_thread / 2; + L->dst_spad_half_size = L->dst_bytes_per_thread / 2; + + const size_t src0_total = n_threads * L->src0_bytes_per_thread; + const size_t src1_total = (kparams->src1_size > 0) + ? kparams->src1_size + : n_threads * L->src1_bytes_per_thread; + const size_t dst_total = n_threads * L->dst_bytes_per_thread; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, src0_total); + VTCM_LAYOUT_ALLOC(off, off_src1, src1_total); + VTCM_LAYOUT_ALLOC(off, off_dst, dst_total); + + L->total_bytes = off; +} + +#endif diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c index 51d39e8d98f5..1fa6ec1bdf1a 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.c +++ b/ggml/src/ggml-hexagon/htp/concat-ops.c @@ -1,9 +1,12 @@ +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hexagon_types.h" #include "hexagon_protos.h" #include "hvx_hexagon_protos.h" -#include "hex-dma.h" +#include "dma-queue.h" #include "htp-vtcm.h" #include "hvx-utils.h" #include "hex-fastdiv.h" @@ -13,6 +16,10 @@ struct htp_concat_context { struct htp_ops_context * octx; uint32_t dim; uint32_t nrows_per_thread; + uint32_t row_start; + uint32_t nrows; + uint32_t elem_start; + uint32_t nelems; struct fastdiv_values div_ne0; struct fastdiv_values div_ne1; struct fastdiv_values div_ne2; @@ -28,13 +35,13 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src0_ne0 = src0->ne[0]; const uint32_t src1_ne0 = src1->ne[0]; - const uint32_t ne1 = dst->ne[1]; - const uint32_t start_i = ith * cctx->nrows_per_thread; - const uint32_t end_i = (start_i + cctx->nrows_per_thread < ne1) ? (start_i + cctx->nrows_per_thread) : ne1; + const uint32_t row_end = cctx->row_start + cctx->nrows; + const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread; + const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end; if (start_i >= end_i) return; - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; uint8_t * spad0_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; uint8_t * spad1_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; @@ -51,21 +58,24 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * const uint32_t spad0_row_bytes = hex_round_up((src0_ne0 + src1_ne0_padded) * sizeof(float), VLEN); uint32_t mu = src1_ne0_padded * spad1_stride; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t i = start_i; i < end_i; i += block_i) { uint32_t current_block_i = (end_i - i < block_i) ? (end_i - i) : block_i; uint32_t src1_width_bytes = current_block_i * sizeof(float); - uint8_t * src1_ptr = (uint8_t *)src1->data + i * src1->nb[1]; - dma_queue_push(q, dma_make_ptr(spad1_base, src1_ptr), spad1_stride, src1->nb[0], src1_width_bytes, src1_ne0); + const dma_addr_t src1_addr = src1->data + i * src1->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], src1_width_bytes, src1_ne0); uint32_t src0_row_bytes = src0_ne0 * sizeof(float); - uint8_t * src0_ptr = (uint8_t *)src0->data + i * src0->nb[1]; - dma_queue_push(q, dma_make_ptr(spad0_base, src0_ptr), spad0_row_bytes, src0->nb[1], src0_row_bytes, current_block_i); + const dma_addr_t src0_addr = src0->data + i * src0->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad0_base, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, current_block_i); - dma_queue_pop(q); // src1 + dma_queue_pop(dma_q); // src1 HVX_Vector * vtcm_tmp = (HVX_Vector *)(spad1_base + src1_ne0_padded * spad1_stride); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); for (uint32_t j = 0; j < src1_ne0_padded; j += 32) { #pragma unroll(4) for (uint32_t ii = 0; ii < current_block_i; ii++) { @@ -75,13 +85,14 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * hvx_vmemu(dst_ptr) = vtcm_tmp[ii]; } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); - dma_queue_pop(q); // src0 + dma_queue_pop(dma_q); // src0 - uint8_t * dst_ptr = (uint8_t *)dst->data + i * dst->nb[1]; - dma_queue_push(q, dma_make_ptr(dst_ptr, spad0_base), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(float), current_block_i); + const dma_addr_t dst_addr = dst->data + i * dst->nb[1]; + dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(float), current_block_i); - dma_queue_pop(q); + dma_queue_pop(dma_q); } } @@ -95,13 +106,13 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src0_ne0 = src0->ne[0]; const uint32_t src1_ne0 = src1->ne[0]; - const uint32_t ne1 = dst->ne[1]; - const uint32_t start_i = ith * cctx->nrows_per_thread; - const uint32_t end_i = (start_i + cctx->nrows_per_thread < ne1) ? (start_i + cctx->nrows_per_thread) : ne1; + const uint32_t row_end = cctx->row_start + cctx->nrows; + const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread; + const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end; if (start_i >= end_i) return; - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; uint8_t * spad0_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; uint8_t * spad1_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; @@ -118,21 +129,24 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * const uint32_t spad0_row_bytes = hex_round_up((src0_ne0 + src1_ne0_padded) * sizeof(__fp16), VLEN); uint32_t mu = src1_ne0_padded * spad1_stride; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t i = start_i; i < end_i; i += block_i) { uint32_t current_block_i = (end_i - i < block_i) ? (end_i - i) : block_i; uint32_t src1_width_bytes = current_block_i * sizeof(__fp16); - uint8_t * src1_ptr = (uint8_t *)src1->data + i * src1->nb[1]; - dma_queue_push(q, dma_make_ptr(spad1_base, src1_ptr), spad1_stride, src1->nb[0], src1_width_bytes, src1_ne0); + const dma_addr_t src1_addr = src1->data + i * src1->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], src1_width_bytes, src1_ne0); uint32_t src0_row_bytes = src0_ne0 * sizeof(__fp16); - uint8_t * src0_ptr = (uint8_t *)src0->data + i * src0->nb[1]; - dma_queue_push(q, dma_make_ptr(spad0_base, src0_ptr), spad0_row_bytes, src0->nb[1], src0_row_bytes, current_block_i); + const dma_addr_t src0_addr = src0->data + i * src0->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad0_base, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, current_block_i); - dma_queue_pop(q); // src1 + dma_queue_pop(dma_q); // src1 HVX_Vector * vtcm_tmp = (HVX_Vector *)(spad1_base + src1_ne0_padded * spad1_stride); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); for (uint32_t j = 0; j < src1_ne0_padded; j += 64) { #pragma unroll(4) for (uint32_t ii = 0; ii < current_block_i; ii++) { @@ -142,13 +156,14 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * hvx_vmemu(dst_ptr) = vtcm_tmp[ii]; } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); - dma_queue_pop(q); // src0 + dma_queue_pop(dma_q); // src0 - uint8_t * dst_ptr = (uint8_t *)dst->data + i * dst->nb[1]; - dma_queue_push(q, dma_make_ptr(dst_ptr, spad0_base), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(__fp16), current_block_i); + const dma_addr_t dst_addr = dst->data + i * dst->nb[1]; + dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(__fp16), current_block_i); - dma_queue_pop(q); + dma_queue_pop(dma_q); } } @@ -164,11 +179,14 @@ static void concat_generic(unsigned int nth, unsigned int ith, void * data) { const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2; const uint32_t ne[4] = {dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]}; - const uint32_t total_elements = ne[0] * ne[1] * ne[2] * ne[3]; - const uint32_t chunk_size = (total_elements + nth - 1) / nth; - const uint32_t start_idx = MIN(ith * chunk_size, total_elements); - const uint32_t end_idx = MIN(start_idx + chunk_size, total_elements); + // Per-device element range aligned to prevent false sharing + const uint32_t elem_start = cctx->elem_start; + const uint32_t nelems = cctx->nelems; + const uint32_t chunk_size = (nelems + nth - 1) / nth; + + const uint32_t start_idx = MIN(elem_start + ith * chunk_size, elem_start + nelems); + const uint32_t end_idx = MIN(start_idx + chunk_size, elem_start + nelems); // Naive scalar element-wise copy for (uint32_t idx = start_idx; idx < end_idx; idx++) { @@ -236,13 +254,28 @@ int op_concat(struct htp_ops_context * octx) { void (*worker_func)(unsigned int, unsigned int, void *) = concat_generic; if (dim == 0 && is_2d && is_src1_transposed && !is_src0_transposed) { - n_threads = MIN(dst->ne[1], n_threads); - if (n_threads < 1) { - n_threads = 1; + const uint32_t total_rows = dst->ne[1]; + const size_t dst_data_row_size = dst->ne[0] * type_size; + uint32_t row_start = 0; + uint32_t nrows = total_rows; + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + cctx.row_start = row_start; + cctx.nrows = nrows; + uint32_t block_i = (type_size == 4) ? 32 : 64; - cctx.nrows_per_thread = hmx_ceil_div(dst->ne[1], n_threads); + cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); // Allocate VTCM uint32_t spad1_stride = block_i * type_size; @@ -270,8 +303,30 @@ int op_concat(struct htp_ops_context * octx) { } else { worker_func = concat_2d_f16_transposed; } + } else { + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + const uint32_t total_elements = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; + uint32_t elem_start = 0; + uint32_t nelems = total_elements; + if (octx->ctx->mdev.count > 1) { + const uint32_t elems_per_chunk = HEX_L2_LINE_SIZE / type_size; + const bool can_split = htp_tensor_mdev_data_aligned(dst) && htp_tensor_is_contiguous(dst, type_size) && !htp_tensor_is_permuted(dst); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elements, can_split ? elems_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + elem_start = range.start; + nelems = range.count; + } + + if (nelems == 0) { + return HTP_STATUS_OK; + } + + cctx.elem_start = elem_start; + cctx.nelems = nelems; } - worker_pool_run_func(octx->ctx->worker_pool, worker_func, &cctx, n_threads); + work_queue_run(octx->ctx->work_queue, worker_func, &cctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index b151b757f413..4453dda3adae 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -16,6 +16,7 @@ #include "htp-ops.h" #include "hvx-utils.h" #include "htp-tensor.h" +#include "htp-fence.h" struct htp_copy_context { struct htp_ops_context * octx; @@ -29,9 +30,49 @@ struct htp_copy_context { uint32_t src0_blocks_per_row; uint32_t dst_blocks_per_row; + uint32_t elem_start; + uint32_t nelem; + uint32_t elem_per_thread; + uint32_t src0_nrows_per_thread; + uint32_t row_start; + uint32_t nrows; + + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02_ne01; + + struct fastdiv_values div_ne0; + struct fastdiv_values div_ne1_ne0; + struct fastdiv_values div_ne2_ne1_ne0; + struct fastdiv_values div_ne00; + struct fastdiv_values div_ne01_ne00; + struct fastdiv_values div_ne02_ne01_ne00; }; +static inline void cpy_dma_sametype_reshape_contig( + dma_queue * dma_q, + dma_addr_t dst, + dma_addr_t src0, + uint32_t total_bytes +) { + if (total_bytes == 0) { + return; + } + + const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE; + while (total_bytes > 0) { + const uint32_t chunk = MIN(total_bytes, max_chunk); + if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1)) { + dma_queue_flush(dma_q); + dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1); + } + dst += chunk; + src0 += chunk; + total_bytes -= chunk; + } + dma_queue_flush(dma_q); +} + #define cpy_preamble \ const struct htp_tensor *src0 = octx->src[0]; \ const struct htp_tensor *dst = octx->dst; \ @@ -54,131 +95,133 @@ struct htp_copy_context { const uint32_t nb0 = dst->nb[0]; \ const uint32_t nb1 = dst->nb[1]; \ const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ - const uint32_t nr = ne01; - -#define DEFINE_CPY_SAMESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ -static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_copy_context * ct = (struct htp_copy_context *) data; \ - struct htp_ops_context * octx = ct->octx; \ - cpy_preamble; \ - const uint32_t dr = ct->src0_nrows_per_thread; \ - const uint32_t ir0 = dr * ith; \ - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; \ - if (ir0 >= nr) return; \ - for (uint32_t i03 = 0; i03 < ne03; i03++) { \ - for (uint32_t i02 = 0; i02 < ne02; i02++) { \ - _Pragma("unroll(4)") \ - for (uint32_t i01 = ir0; i01 < ir1; i01++) { \ - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; \ - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ - hex_l2fetch(src0_ptr, ne00 * ELEM_SIZE, nb01, 2); \ - hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \ - } \ - } \ - } \ + const uint32_t nb3 = dst->nb[3]; + +#define DEFINE_CPY_SAMESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ +static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_copy_context * ct = (struct htp_copy_context *) data; \ + struct htp_ops_context * octx = ct->octx; \ + cpy_preamble; \ + const uint32_t dr = ct->src0_nrows_per_thread; \ + const uint32_t ir0 = ct->row_start + dr * ith; \ + const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \ + if (ir0 >= ir1) return; \ + const bool contiguous = htp_tensor_is_contiguous(src0, ELEM_SIZE) && htp_tensor_is_contiguous(dst, ELEM_SIZE); \ + if (contiguous) { \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + dma_addr_t dst_addr = dst->data + ir0 * ne00 * ELEM_SIZE; \ + dma_addr_t src0_addr = src0->data + ir0 * ne00 * ELEM_SIZE; \ + cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (ir1 - ir0) * ne00 * ELEM_SIZE); \ + return; \ + } \ + const uint32_t ne02_ne01 = ne02 * ne01; \ + uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); \ + uint32_t rem = ir0 - i03 * ne02_ne01; \ + uint32_t i02 = fastdiv(rem, &ct->div_ne01); \ + uint32_t i01 = rem - i02 * ne01; \ + uint8_t * dst_ptr = (uint8_t *) dst->data + i01*nb1 + i02*nb2 + i03*nb3; \ + uint8_t * src0_ptr = (uint8_t *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ + for (uint32_t r = ir0; r < ir1; r++) { \ + hex_l2fetch(src0_ptr, ne00 * ELEM_SIZE, nb01, 2); \ + hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \ + dst_ptr += nb1; \ + src0_ptr += nb01; \ + if (++i01 == ne01) { \ + i01 = 0; \ + if (++i02 == ne02) { \ + i02 = 0; \ + i03++; \ + } \ + dst_ptr = (uint8_t *) dst->data + i02*nb2 + i03*nb3; \ + src0_ptr = (uint8_t *) src0->data + i02*nb02 + i03*nb03; \ + } \ + } \ } DEFINE_CPY_SAMESHAPE(f32, float, 4) DEFINE_CPY_SAMESHAPE(f16, __fp16, 2) -#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ -static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_copy_context * ct = (struct htp_copy_context *) data; \ - struct htp_ops_context * octx = ct->octx; \ - cpy_preamble; \ - const uint32_t dr = ct->src0_nrows_per_thread; \ - const uint32_t ir0 = dr * ith; \ - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; \ - if (ir0 >= nr) return; \ - const bool src0_contig = (nb00 == ELEM_SIZE) && \ - (nb01 == ne00 * nb00) && \ - (nb02 == ne01 * nb01) && \ - (nb03 == ne02 * nb02); \ - const bool dst_contig = (nb0 == ELEM_SIZE) && \ - (nb1 == ne0 * nb0) && \ - (nb2 == ne1 * nb1) && \ - (nb3 == ne2 * nb2); \ - if (src0_contig && dst_contig) { \ - for (int64_t i03 = 0; i03 < ne03; i03++) { \ - for (int64_t i02 = 0; i02 < ne02; i02++) { \ - uint8_t * src_ptr = (uint8_t *) src0->data + i03*nb03 + i02*nb02 + ir0*nb01; \ - uint32_t flat = ((i03*ne02 + i02)*ne01 + ir0) * ne00; \ - uint8_t * dst_ptr = (uint8_t *) dst->data + flat * ELEM_SIZE; \ - hvx_copy_uu(dst_ptr, src_ptr, (ir1 - ir0) * ne00, ELEM_SIZE); \ - } \ - } \ - return; \ - } \ - const bool reshape_flat_fast = (ne03 == 1 && ne2 == 1 && ne3 == 1) && \ - (ne0 == ne00 * ne01) && (ne1 == ne02) && \ - (nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \ - if (reshape_flat_fast) { \ - for (uint32_t i02 = 0; i02 < ne02; i02++) { \ - for (uint32_t i01 = ir0; i01 < ir1; i01++) { \ - uint8_t * src0_ptr = (uint8_t *) src0->data + i01 * nb01 + i02 * nb02; \ - uint8_t * dst_ptr = (uint8_t *) dst->data + i01 * ne00 * ELEM_SIZE + i02 * nb1; \ - hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \ - } \ - } \ - return; \ - } \ - int64_t k10 = 0; \ - int64_t i11 = 0; \ - int64_t i12 = 0; \ - int64_t i13 = 0; \ - const int64_t nk00 = ct->src0_blocks_per_row; \ - const int64_t nk0 = ct->dst_blocks_per_row; \ - for (int64_t i03 = 0; i03 < ne03; i03++) { \ - for (int64_t i02 = 0; i02 < ne02; i02++) { \ - k10 += nk00 * ir0; \ - while (k10 >= nk0) { \ - k10 -= nk0; \ - if (++i11 == ne1) { \ - i11 = 0; \ - if (++i12 == ne2) { \ - i12 = 0; \ - if (++i13 == ne3) { \ - i13 = 0; \ - } \ - } \ - } \ - } \ - for (int64_t i01 = ir0; i01 < ir1; i01++) { \ - for (int64_t k00 = 0; k00 < nk00; k00++) { \ - const char * src0_ptr = ((char *) src0->data + k00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); \ - char * dst_ptr = ((char *) dst->data + k10*nb0 + i11*nb1 + i12*nb2 + i13*nb3); \ - memcpy(dst_ptr, src0_ptr, ELEM_SIZE); \ - if (++k10 == nk0) { \ - k10 = 0; \ - if (++i11 == ne1) { \ - i11 = 0; \ - if (++i12 == ne2) { \ - i12 = 0; \ - if (++i13 == ne3) { \ - i13 = 0; \ - } \ - } \ - } \ - } \ - } \ - } \ - k10 += nk00 * (ne01 - ir1); \ - while (k10 >= nk0) { \ - k10 -= nk0; \ - if (++i11 == ne1) { \ - i11 = 0; \ - if (++i12 == ne2) { \ - i12 = 0; \ - if (++i13 == ne3) { \ - i13 = 0; \ - } \ - } \ - } \ - } \ - } \ - } \ +#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ +static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_copy_context * ct = (struct htp_copy_context *) data; \ + struct htp_ops_context * octx = ct->octx; \ + cpy_preamble; \ + const uint32_t th_nelem = ct->elem_per_thread; \ + const uint32_t th_start = ct->elem_start + ith * th_nelem; \ + const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \ + if (th_start >= th_end) return; \ + \ + if (htp_tensor_is_contiguous(src0, ELEM_SIZE) && htp_tensor_is_contiguous(dst, ELEM_SIZE)) { \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + dma_addr_t dst_addr = dst->data + th_start * ELEM_SIZE; \ + dma_addr_t src0_addr = src0->data + th_start * ELEM_SIZE; \ + cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (th_end - th_start) * ELEM_SIZE); \ + return; \ + } \ + \ + const uint32_t ne01_ne00 = ne01 * ne00; \ + const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \ + const uint32_t ne1_ne0 = ne1 * ne0; \ + const uint32_t ne2_ne1_ne0 = ne2 * ne1_ne0; \ + \ + uint32_t e = th_start; \ + uint32_t i13 = fastdiv(e, &ct->div_ne2_ne1_ne0); \ + uint32_t rem = e - i13 * ne2_ne1_ne0; \ + uint32_t i12 = fastdiv(rem, &ct->div_ne1_ne0); \ + uint32_t rem2 = rem - i12 * ne1_ne0; \ + uint32_t i11 = fastdiv(rem2, &ct->div_ne0); \ + uint32_t i10 = rem2 - i11 * ne0; \ + \ + uint32_t i03 = fastdiv(e, &ct->div_ne02_ne01_ne00); \ + uint32_t rem_s = e - i03 * ne02_ne01_ne00; \ + uint32_t i02 = fastdiv(rem_s, &ct->div_ne01_ne00); \ + uint32_t rem2_s = rem_s - i02 * ne01_ne00; \ + uint32_t i01 = fastdiv(rem2_s, &ct->div_ne00); \ + uint32_t i00 = rem2_s - i01 * ne00; \ + \ + char * dst_ptr = (char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \ + const char * src0_ptr = (const char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \ + \ + const bool rows_contig = (nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \ + \ + while (e < th_end) { \ + uint32_t run = 1; \ + if (rows_contig) { \ + run = MIN(MIN(ne00 - i00, ne0 - i10), th_end - e); \ + hvx_copy_uu((uint8_t *) dst_ptr, (const uint8_t *) src0_ptr, run, ELEM_SIZE); \ + } else { \ + *((ELEM_TYPE *) dst_ptr) = *((const ELEM_TYPE *) src0_ptr); \ + } \ + e += run; \ + \ + dst_ptr += run * nb0; \ + i10 += run; \ + if (i10 == ne0) { \ + i10 = 0; \ + if (++i11 == ne1) { \ + i11 = 0; \ + if (++i12 == ne2) { \ + i12 = 0; \ + i13++; \ + } \ + } \ + dst_ptr = (char *) dst->data + i11*nb1 + i12*nb2 + i13*nb3; \ + } \ + \ + src0_ptr += run * nb00; \ + i00 += run; \ + if (i00 == ne00) { \ + i00 = 0; \ + if (++i01 == ne01) { \ + i01 = 0; \ + if (++i02 == ne02) { \ + i02 = 0; \ + i03++; \ + } \ + } \ + src0_ptr = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ + } \ + } \ } DEFINE_CPY_RESHAPE(f32, float, 4) @@ -189,22 +232,33 @@ static void cpy_thread_f16_f32_sameshape(unsigned int nth, unsigned int ith, voi struct htp_ops_context * octx = ct->octx; cpy_preamble; - // parallelize by src0 rows const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; - if (ir0 >= nr) return; - - // copy by rows - for (uint32_t i03 = 0; i03 < ne03; i03++) { - for (uint32_t i02 = 0; i02 < ne02; i02++) { - #pragma unroll(2) - for (uint32_t i01 = ir0; i01 < ir1; i01++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2); - hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00); + const uint32_t ir0 = ct->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); + if (ir0 >= ir1) return; + + const uint32_t ne02_ne01 = ne02 * ne01; + uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); + uint32_t rem = ir0 - i03 * ne02_ne01; + uint32_t i02 = fastdiv(rem, &ct->div_ne01); + uint32_t i01 = rem - i02 * ne01; + + uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; + uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + + for (uint32_t r = ir0; r < ir1; r++) { + hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2); + hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00); + dst_ptr += nb1; + src0_ptr += nb01; + if (++i01 == ne01) { + i01 = 0; + if (++i02 == ne02) { + i02 = 0; + i03++; } + dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; + src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; } } } @@ -214,23 +268,55 @@ static void cpy_thread_f32_f16_sameshape(unsigned int nth, unsigned int ith, voi struct htp_ops_context * octx = ct->octx; cpy_preamble; - // parallelize by src0 rows const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; - if (ir0 >= nr) return; - - // copy by rows - for (uint32_t i03 = 0; i03 < ne03; i03++) { - for (uint32_t i02 = 0; i02 < ne02; i02++) { - #pragma unroll(2) - for (uint32_t i01 = ir0; i01 < ir1; i01++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2); - hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00); + const uint32_t ir0 = ct->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); + if (ir0 >= ir1) return; + + const uint32_t ne02_ne01 = ne02 * ne01; + uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); + uint32_t rem = ir0 - i03 * ne02_ne01; + uint32_t i02 = fastdiv(rem, &ct->div_ne01); + uint32_t i01 = rem - i02 * ne01; + + uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; + uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + + for (uint32_t r = ir0; r < ir1; r++) { + hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2); + hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00); + dst_ptr += nb1; + src0_ptr += nb01; + if (++i01 == ne01) { + i01 = 0; + if (++i02 == ne02) { + i02 = 0; + i03++; } + dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; + src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; + } + } +} + +static inline void cpy_dma_push_2d_chunked( + dma_queue * dma_q, + dma_addr_t dst, + dma_addr_t src, + size_t dst_stride, + size_t src_stride, + size_t row_size, + uint32_t nrows +) { + while (nrows > 0) { + const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS); + if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) { + dma_queue_flush(dma_q); + dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows); } + dst += cur_rows * dst_stride; + src += cur_rows * src_stride; + nrows -= cur_rows; } } @@ -243,36 +329,40 @@ static inline void cpy_dma_sametype_sameshape( uint32_t nb01, uint32_t nb02, uint32_t nb03, uint32_t nb1, uint32_t nb2, uint32_t nb3 ) { + const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size); + + dma_queue * dma_q = octx->ctx->dma[0]; + + if (contiguous) { + cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03); + return; + } + const bool contiguous_outer = (ne02 == 1 || (nb02 == ne01 * nb01 && nb2 == ne01 * nb1)) && (ne03 == 1 || (nb03 == ne02 * nb02 && nb3 == ne02 * nb2)); - dma_queue * q = octx->ctx->dma[0]; - if (contiguous_outer) { - dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03); - dma_queue_pop(q); + uint32_t total_rows = ne01 * ne02 * ne03; + cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows); + dma_queue_flush(dma_q); return; } for (uint32_t i03 = 0; i03 < ne03; i03++) { for (uint32_t i02 = 0; i02 < ne02; i02++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; - if (!dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01)) { - dma_queue_flush(q); - dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01); - } + dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3; + dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03; + cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01); } } - dma_queue_flush(q); + dma_queue_flush(dma_q); } -int op_cpy(struct htp_ops_context * octx) { +static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) { cpy_preamble; - - const uint32_t n_threads = MIN(nr, octx->n_threads); + *use_dma = false; struct htp_copy_context ct; ct.octx = octx; @@ -291,64 +381,141 @@ int op_cpy(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; - } - const bool sametype = (src0->type == dst->type); - const bool transposed = (nb00 > nb01) || (nb0 > nb1); + const bool transposed = (nb00 > nb01) || (nb0 > nb1) || + (nb00 != ct.src0_type_size) || (nb0 != ct.dst_type_size) || + (nb01 < ne00 * ct.src0_type_size) || (nb1 < ne0 * ct.dst_type_size); const bool sameshape = !transposed && (ne00 == ne0 && ne01 == ne1 && ne02 == ne2 && ne03 == ne3); - ct.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads; + const uint32_t n_threads = octx->n_threads; - worker_callback_t copy_fun = NULL; - bool use_dma = false; + const bool src_is_contiguous = htp_tensor_is_contiguous(src0, ct.src0_type_size); + const bool dst_is_contiguous = htp_tensor_is_contiguous(dst, ct.dst_type_size); - if (sametype && sameshape) { - use_dma = true; - } else if (sameshape) { - /**/ if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) - copy_fun = cpy_thread_f16_f32_sameshape; - else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) - copy_fun = cpy_thread_f32_f16_sameshape; - else + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + if (!sametype) { return HTP_STATUS_NO_SUPPORT; - } else if (sametype) { - if (src0->type == HTP_TYPE_F32) { - copy_fun = cpy_thread_f32_reshape; + } + if (!sameshape && !(src_is_contiguous && dst_is_contiguous && octx->ctx->mdev.count <= 1)) { + return HTP_STATUS_NO_SUPPORT; + } + } + + if (sameshape) { + const uint32_t total_rows = ne01 * ne02 * ne03; + const uint32_t row_size = ne00 * ct.dst_type_size; + + ct.div_ne01 = init_fastdiv_values(ne01); + ct.div_ne02_ne01 = init_fastdiv_values(ne02 * ne01); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, can_split ? rows_per_chunk : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + ct.row_start = row_start; + ct.nrows = nrows; + ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + + if (sametype && (octx->ctx->mdev.count <= 1 || htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { + if (octx->ctx->mdev.idx == 0) { + *use_dma = true; + cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3); + } } else { - copy_fun = cpy_thread_f16_reshape; + work_queue_func_t copy_fun = NULL; + if (sametype) { + copy_fun = (src0->type == HTP_TYPE_F32) ? cpy_thread_f32_sameshape : cpy_thread_f16_sameshape; + } else if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) { + copy_fun = cpy_thread_f16_f32_sameshape; + } else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) { + copy_fun = cpy_thread_f32_f16_sameshape; + } else { + return HTP_STATUS_NO_SUPPORT; + } + work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); + } + } else if (sametype) { + const uint32_t total_elems = ne0 * ne1 * ne2 * ne3; + const uint32_t elems_per_line = (ct.dst_type_size == 4) ? 32 : 64; + + if (octx->ctx->mdev.count <= 1 && dst_is_contiguous && src_is_contiguous) { + *use_dma = true; + cpy_dma_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, total_elems * ct.dst_type_size); + return HTP_STATUS_OK; + } + + ct.div_ne0 = init_fastdiv_values(ne0); + ct.div_ne1_ne0 = init_fastdiv_values(ne1 * ne0); + ct.div_ne2_ne1_ne0 = init_fastdiv_values(ne2 * ne1 * ne0); + ct.div_ne00 = init_fastdiv_values(ne00); + ct.div_ne01_ne00 = init_fastdiv_values(ne01 * ne00); + ct.div_ne02_ne01_ne00 = init_fastdiv_values(ne02 * ne01 * ne00); + + uint32_t elem_start = 0; + uint32_t nelem = total_elems; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elems, can_split ? elems_per_line : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + elem_start = range.start; + nelem = range.count; + } + + if (nelem == 0) { + return HTP_STATUS_OK; } + + ct.elem_start = elem_start; + ct.nelem = nelem; + ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div); + + work_queue_func_t copy_fun = (src0->type == HTP_TYPE_F32) ? cpy_thread_f32_reshape : cpy_thread_f16_reshape; + work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); } else { return HTP_STATUS_NO_SUPPORT; } - FARF(HIGH, "cpy-%s-%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_dma=%d n_threads %u\n", - src0->type == HTP_TYPE_F32 ? "f32" : "f16", dst->type == HTP_TYPE_F32 ? "f32" : "f16", - ne00, ne01, ne02, ne03, ne0, ne1, ne2, ne3, use_dma, n_threads); + return HTP_STATUS_OK; +} - if (use_dma) { - cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3); - } else { - worker_pool_run_func(octx->ctx->worker_pool, copy_fun, &ct, n_threads); - } +int op_cpy(struct htp_ops_context * octx) { + bool use_dma = false; + int status = exec_cpy(octx, &use_dma); - const struct htp_tensor *sync = octx->src[1]; - if (sync && (sync->flags & HTP_TENSOR_FENCE)) { + htp_ops_context_set_status(octx, status); + + if (octx->op == HTP_OP_CPY_FENCE) { if (!use_dma) { - // htp_tensor_flush_all(octx->ctx, octx->dsts, 1); - qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + htp_flush_dirty_ranges(octx->ctx); } - atomic_uint * sync_fence = (atomic_uint *) sync->data; - const uint32_t seq = (uint32_t) octx->op_params[0]; + htp_mdev_group_barrier(octx); - atomic_store(&sync_fence[0], seq); - asm volatile ("syncht" : : : "memory"); - Q6_dccleaninva_A((void *) sync_fence); + if (octx->ctx->mdev.idx == 0) { + const struct htp_tensor * sync = octx->src[1]; + if (htp_tensor_is_extended(sync)) { + return HTP_STATUS_NO_SUPPORT; + } + const uint32_t seq = (uint32_t) octx->op_params[0]; + atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data; + htp_fence_write(sync_fence, seq, octx->status); - FARF(HIGH, "ggml-hex: sync-release : fence %p seq %u\n", sync_fence, seq); + FARF(HIGH, "ggml-hex: sync-release : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status); + } } - return HTP_STATUS_OK; + return octx->status; } diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.c index 2d45c39f23b5..eaab7d7e516c 100644 --- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c @@ -7,35 +7,37 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" #include "hvx-types.h" #include "hvx-utils.h" -#include "hex-dma.h" +#include "dma-queue.h" #define htp_cumsum_tensors_preamble \ const struct htp_tensor * restrict src0 = octx->src[0]; \ const struct htp_tensor * restrict dst = octx->dst; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ + \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb00 = src0->nb[0]; \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb0 = dst->nb[0]; \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; struct htp_cumsum_context { @@ -46,13 +48,14 @@ struct htp_cumsum_context { size_t dst_row_size_aligned; uint32_t rows_per_thread; uint32_t total_rows; + uint32_t row_start; }; #define htp_cumsum_preamble \ struct htp_cumsum_context * cctx = (struct htp_cumsum_context *) data; \ struct htp_ops_context * octx = cctx->octx; \ htp_cumsum_tensors_preamble; \ - dma_queue * dma_queue = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; // --------------------------------------------------------------------------- // HVX prefix scan helpers @@ -116,11 +119,8 @@ static inline void hvx_cumsum_row_f32(const float * restrict src, float * restri static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t ir0 = cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); if (ir0 >= ir1) { return; @@ -131,49 +131,51 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat const size_t src_row_size_aligned = cctx->src_row_size_aligned; const size_t dst_row_size_aligned = cctx->dst_row_size_aligned; - const uint8_t * src_data = (const uint8_t *) src0->data; - uint8_t * dst_data = (uint8_t *) dst->data; + const dma_addr_t src_data = src0->data; + const dma_addr_t dst_data = dst->data; uint8_t * src_spad = octx->src0_spad.data + (ith * src_row_size_aligned * 2); uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned * 2); for (uint32_t ir = ir0, spad_idx = 0; ir < ir1 && spad_idx < 2; ir++, spad_idx++) { // Dummy dst writeback to establish queue ordering - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(dst_data, dst_spad + (spad_idx * dst_row_size_aligned)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src_spad + (spad_idx * src_row_size_aligned), - src_data + (ir * src_row_size)), - src_row_size_aligned, src_row_size, 1); + dma_queue_push(dma_q, + dma_make_data(dst_data, dst_spad + (spad_idx * dst_row_size_aligned)), + dst_row_size, dst_row_size_aligned, dst_row_size, 0); + + dma_queue_push(dma_q, + dma_make_data(src_spad + (spad_idx * src_row_size_aligned), + src_data + (ir * src_row_size)), + src_row_size_aligned, src_row_size, src_row_size, 1); } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = ir0; ir < ir1; ir++) { - float * dst_spad_row = (float *) dma_queue_pop(dma_queue).src; - float * src_spad_row = (float *) dma_queue_pop(dma_queue).dst; + float * dst_spad_row = (float *) dma_queue_pop(dma_q).src; + float * src_spad_row = (float *) dma_queue_pop(dma_q).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); hvx_cumsum_row_f32(src_spad_row, dst_spad_row, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(dst_data + (ir * dst_row_size), (uint8_t *) dst_spad_row), - dst_row_size, dst_row_size_aligned, 1); + dma_queue_push(dma_q, + dma_make_data(dst_data + (ir * dst_row_size), dst_spad_row), + dst_row_size, dst_row_size_aligned, dst_row_size, 1); const uint32_t next_row = ir + 2; if (next_row < ir1) { - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr((uint8_t *) src_spad_row, src_data + (next_row * src_row_size)), - src_row_size_aligned, src_row_size, 1); + dma_queue_push(dma_q, + dma_make_data(src_spad_row, src_data + (next_row * src_row_size)), + src_row_size_aligned, src_row_size, src_row_size, 1); } } - dma_queue_flush(dma_queue); - t2 = HAP_perf_get_qtimer_count(); + dma_queue_flush(dma_q); - FARF(HIGH, "cumsum-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "cumsum-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } // --------------------------------------------------------------------------- @@ -183,14 +185,14 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ir0 = cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); for (uint32_t ir = ir0; ir < ir1; ir++) { const float * restrict src_row = (const float *) (src_data + ir * cctx->src_row_size); @@ -198,24 +200,36 @@ static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { hvx_cumsum_row_f32(src_row, dst_row, ne00); } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); - FARF(HIGH, "cumsum-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "cumsum-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } int op_cumsum_f32(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { return HTP_STATUS_OK; } - const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_rows); + const uint32_t n_threads = octx->n_threads; const size_t src_row_size = src0->nb[1]; const size_t dst_row_size = dst->nb[1]; @@ -240,14 +254,18 @@ int op_cumsum_f32(struct htp_ops_context * octx) { .dst_row_size = dst_row_size, .src_row_size_aligned = src_row_size_aligned, .dst_row_size_aligned = dst_row_size_aligned, - .rows_per_thread = (total_rows + n_threads - 1) / n_threads, - .total_rows = total_rows, + .rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_rows = nrows, + .row_start = row_start, }; if (octx->ctx->vtcm_size < spad_per_thread * n_threads) { - worker_pool_run_func(octx->ctx->worker_pool, cumsum_thread_f32, &cctx, n_threads); + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads); } else { - worker_pool_run_func(octx->ctx->worker_pool, cumsum_thread_f32_dma, &cctx, n_threads); + work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.c b/ggml/src/ggml-hexagon/htp/diag-ops.c index 9b3194d90846..162214d3e46f 100644 --- a/ggml/src/ggml-hexagon/htp/diag-ops.c +++ b/ggml/src/ggml-hexagon/htp/diag-ops.c @@ -5,27 +5,30 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-types.h" #include "hex-utils.h" #include "hvx-copy.h" -#include "hex-dma.h" +#include "dma-queue.h" #define htp_diag_tensors_preamble \ const struct htp_tensor * restrict src0 = octx->src[0]; \ const struct htp_tensor * restrict dst = octx->dst; \ - \ - const uint32_t ne02 = src0->ne[2]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ + \ + const uint32_t ne02 = src0->ne[2]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; struct htp_diag_context { @@ -36,6 +39,7 @@ struct htp_diag_context { size_t dst_row_size_aligned; uint32_t batches_per_thread; uint32_t total_batches; + uint32_t batch_start; }; #define htp_diag_preamble \ @@ -55,13 +59,10 @@ static inline void hvx_diag_row_f32(const float * restrict src, float * restrict static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { htp_diag_preamble; - dma_queue * dma_queue = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t ib0 = dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); if (ib0 >= ib1) { return; @@ -72,47 +73,48 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) const size_t src_batch_size_aligned = dctx->src_batch_size_aligned; const size_t dst_row_size_aligned = dctx->dst_row_size_aligned; - const uint8_t * src_data = (const uint8_t *) src0->data; - uint8_t * dst_data = (uint8_t *) dst->data; + const dma_addr_t src_data = src0->data; + const dma_addr_t dst_data = dst->data; // 1 src buffer + 1 dst row buffer per thread in VTCM uint8_t * src_spad = octx->src0_spad.data + (ith * src_batch_size_aligned); uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ib = ib0; ib < ib1; ib++) { const uint32_t i3 = ib / ne02; const uint32_t i2 = ib % ne02; - const uint8_t * src_batch = src_data + i3 * nb03 + i2 * nb02; + const dma_addr_t src_batch = src_data + i3 * nb03 + i2 * nb02; // Fetch source vector into VTCM - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src_spad, src_batch), - src_batch_size_aligned, src_batch_size, 1); - dma_queue_flush(dma_queue); + dma_queue_push(dma_q, + dma_make_data(src_spad, src_batch), + src_batch_size_aligned, src_batch_size, src_batch_size, 1); + dma_queue_flush(dma_q); const float * src_spad_f32 = (const float *) src_spad; float * dst_spad_f32 = (float *) dst_spad; for (uint32_t i1 = 0; i1 < ne1; i1++) { // Compute row in VTCM + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1)); hvx_diag_row_f32(src_spad_f32, dst_spad_f32, i1, ne0); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1)); // Write completed row back to DDR - uint8_t * dst_row = dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1; - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(dst_row, dst_spad), - dst_row_size, dst_row_size_aligned, 1); - dma_queue_flush(dma_queue); + const dma_addr_t dst_row = dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1; + dma_queue_push(dma_q, + dma_make_data(dst_row, dst_spad), + dst_row_size, dst_row_size_aligned, dst_row_size, 1); + dma_queue_flush(dma_q); } } - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "diag-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "diag-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } // --------------------------------------------------------------------------- @@ -122,14 +124,14 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) { htp_diag_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ib0 = dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0); for (uint32_t ib = ib0; ib < ib1; ib++) { const uint32_t i3 = ib / ne02; @@ -143,24 +145,48 @@ static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) { } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0); - FARF(HIGH, "diag-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "diag-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } int op_diag_f32(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + const uint32_t total_batches = src0->ne[2] * src0->ne[3]; + const size_t dst_batch_size = dst->ne[1] * dst->nb[1]; + + uint32_t batch_start = 0; + uint32_t nbatches = total_batches; + + if (octx->ctx->mdev.count > 1) { + bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !htp_tensor_is_permuted(dst); + uint32_t batches_per_chunk = 1; + if (can_split) { + if (dst->ne[2] > 1 && (dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && + (dst->ne[3] <= 1 || (dst->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) { + batches_per_chunk = 1; + } else if (dst->nb[2] == dst_batch_size && + (dst->ne[3] <= 1 || dst->nb[3] == dst->nb[2] * dst->ne[2])) { + batches_per_chunk = (dst_batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_batch_size, HEX_L2_LINE_SIZE)) : 1; + } else { + can_split = false; + } + } + + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + batch_start = range.start; + nbatches = range.count; + } + + if (nbatches == 0) { return HTP_STATUS_OK; } - const uint32_t total_batches = src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_batches); + const uint32_t n_threads = octx->n_threads; const size_t src_batch_size = src0->ne[0] * sizeof(float); const size_t dst_row_size = dst->ne[0] * sizeof(float); @@ -185,14 +211,18 @@ int op_diag_f32(struct htp_ops_context * octx) { .dst_row_size = dst_row_size, .src_batch_size_aligned = src_batch_size_aligned, .dst_row_size_aligned = dst_row_size_aligned, - .batches_per_thread = (total_batches + n_threads - 1) / n_threads, - .total_batches = total_batches, + .batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div), + .total_batches = nbatches, + .batch_start = batch_start, }; if (octx->ctx->vtcm_size < spad_per_thread * n_threads) { - worker_pool_run_func(octx->ctx->worker_pool, diag_thread_f32, &dctx, n_threads); + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads); } else { - worker_pool_run_func(octx->ctx->worker_pool, diag_thread_f32_dma, &dctx, n_threads); + work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.c b/ggml/src/ggml-hexagon/htp/dma-queue.c index 4beded1de508..464e4b849de8 100644 --- a/ggml/src/ggml-hexagon/htp/dma-queue.c +++ b/ggml/src/ggml-hexagon/htp/dma-queue.c @@ -22,58 +22,69 @@ static inline uintptr_t align_up(uintptr_t addr, size_t align) { return (addr + align - 1) & ~(align - 1); } -size_t dma_queue_sizeof(size_t capacity) { +static inline size_t dma_ring_sizeof(size_t capacity) { capacity = pow2_ceil(capacity); - size_t size_q = sizeof(dma_queue); - size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE); size_t size_r = sizeof(dma_ring); - size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE); + size_t offset_desc = align_up(size_r, HEX_L2_LINE_SIZE); size_t size_desc = capacity * sizeof(dma_descriptor_2d); - size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE); - size_t size_dptr = capacity * sizeof(dma_ptr); + size_t offset_data = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE); + size_t size_data = capacity * sizeof(dma_data); - return offset_dptr + size_dptr; + return offset_data + size_data; } -size_t dma_queue_alignof(void) { - return HEX_L2_LINE_SIZE; -} - -dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace) { +static inline dma_ring * dma_ring_init(void * ptr, size_t capacity, struct htp_thread_trace * trace) { capacity = pow2_ceil(capacity); - size_t size_q = sizeof(dma_queue); - size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE); size_t size_r = sizeof(dma_ring); - size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE); + size_t offset_desc = align_up(size_r, HEX_L2_LINE_SIZE); size_t size_desc = capacity * sizeof(dma_descriptor_2d); - size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE); - size_t size_dptr = capacity * sizeof(dma_ptr); - - size_t total_size = offset_dptr + size_dptr; - memset(ptr, 0, total_size); - - dma_queue * q = (dma_queue *) ptr; - dma_ring * r = (dma_ring *) ((uintptr_t) ptr + offset_r); - - q->ring = r; - q->nocache = 0; - q->alias = false; + size_t offset_data = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE); + dma_ring * r = (dma_ring *) ptr; r->trace = trace; - r->vtcm_base = vtcm_base; - r->vtcm_end = vtcm_base + vtcm_size; r->capacity = capacity; r->idx_mask = capacity - 1; r->push_idx = 0; r->pop_idx = 0; + r->desc = (dma_descriptor_2d *) ((uintptr_t) ptr + offset_desc); + r->data = (dma_data *) ((uintptr_t) ptr + offset_data); + r->tail = &r->desc[capacity - 1]; + + return r; +} + +size_t dma_queue_sizeof(size_t capacity) { + size_t size_q = sizeof(dma_queue); + size_t offset_r0 = align_up(size_q, HEX_L2_LINE_SIZE); + size_t size_r0 = dma_ring_sizeof(capacity); + size_t offset_r1 = align_up(offset_r0 + size_r0, HEX_L2_LINE_SIZE); + size_t size_r1 = dma_ring_sizeof(DMA_FALLBACK_CAPACITY); + + return offset_r1 + size_r1; +} + +size_t dma_queue_alignof(void) { + return HEX_L2_LINE_SIZE; +} + +dma_queue_t dma_queue_init(void * ptr, size_t capacity, struct htp_thread_trace * trace) { + size_t total_size = dma_queue_sizeof(capacity); + memset(ptr, 0, total_size); + + dma_queue * q = (dma_queue *) ptr; + + size_t size_q = sizeof(dma_queue); + size_t offset_r0 = align_up(size_q, HEX_L2_LINE_SIZE); + size_t size_r0 = dma_ring_sizeof(capacity); + size_t offset_r1 = align_up(offset_r0 + size_r0, HEX_L2_LINE_SIZE); - r->desc = (dma_descriptor_2d *) ((uintptr_t) ptr + offset_desc); - r->dptr = (dma_ptr *) ((uintptr_t) ptr + offset_dptr); - r->tail = &r->desc[capacity - 1]; + q->ring0 = dma_ring_init((void *) ((uintptr_t) ptr + offset_r0), capacity, trace); + q->ring1 = dma_ring_init((void *) ((uintptr_t) ptr + offset_r1), DMA_FALLBACK_CAPACITY, trace); + q->alias = false; - FARF(HIGH, "dma-queue: capacity %u, unified memory size %zu\n", capacity, total_size); + FARF(HIGH, "dma-queue: capacity %u, unified memory size %zu\n", (unsigned) capacity, total_size); return q; } @@ -86,13 +97,13 @@ size_t dma_queue_alias_sizeof(void) { return sizeof(dma_queue); } -dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache) { +dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q) { dma_queue * q = (dma_queue *) ptr; memset(q, 0, sizeof(dma_queue)); - q->ring = main_q->ring; - q->nocache = nocache; - q->alias = true; + q->ring0 = main_q->ring0; + q->ring1 = main_q->ring1; + q->alias = true; return q; } @@ -101,4 +112,101 @@ void dma_queue_alias_free(dma_queue_t q) { (void) q; } +bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + dma_ring * r0 = q->ring0; + dma_ring * r1 = q->ring1; + + if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) { + return false; + } + + r1->tail = r0->tail; + + size_t rem_rows = nrows; + dma_addr_t cur_dst = ddata.dst; + dma_addr_t cur_src = ddata.src; + + while (rem_rows > 0) { + const uint32_t cur_rows = MIN(rem_rows, DMA_MAX_NROWS); + dma_data cur_data = dma_make_data(cur_dst, cur_src); + if (!dma_ring_push_single_2d(r1, cur_data, dst_stride, src_stride, row_size, cur_rows)) { + dma_ring_flush(r1); + dma_ring_push_single_2d(r1, cur_data, dst_stride, src_stride, row_size, cur_rows); + } + cur_dst += cur_rows * dst_stride; + cur_src += cur_rows * src_stride; + rem_rows -= cur_rows; + } + + dma_ring_flush(r1); + r0->tail = r1->tail; + + return dma_ring_push_single_2d(r0, ddata, 0, 0, 0, /*nrows=*/ 0); +} + +bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total) { + dma_ring * r0 = q->ring0; + dma_ring * r1 = q->ring1; + + if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) { + return false; + } + + r1->tail = r0->tail; + + size_t rem_bytes = total; + dma_addr_t cur_dst = ddata.dst; + dma_addr_t cur_src = ddata.src; + + while (rem_bytes > 0) { + const uint32_t cur_bytes = MIN(rem_bytes, DMA_SAFE_CHUNK_SIZE); + dma_data cur_data = dma_make_data(cur_dst, cur_src); + if (!dma_ring_push_single_1d(r1, cur_data, cur_bytes)) { + dma_ring_flush(r1); + dma_ring_push_single_1d(r1, cur_data, cur_bytes); + } + cur_dst += cur_bytes; + cur_src += cur_bytes; + rem_bytes -= cur_bytes; + } + + dma_ring_flush(r1); + r0->tail = r1->tail; + + return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0); +} + +#if __HVX_ARCH__ < 75 + +bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + dma_ring * r0 = q->ring0; + dma_ring * r1 = q->ring1; + + if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) { + return false; + } + + r1->tail = r0->tail; + + size_t rem_rows = nrows; + dma_addr_t cur_dst = ddata.dst; + dma_addr_t cur_src = ddata.src; + + while (rem_rows > 0) { + dma_data cur_data = dma_make_data(cur_dst, cur_src); + if (!dma_ring_push_single_1d(r1, cur_data, row_size)) { + dma_ring_flush(r1); + dma_ring_push_single_1d(r1, cur_data, row_size); + } + cur_dst += dst_stride; + cur_src += src_stride; + rem_rows -= 1; + } + + dma_ring_flush(r1); + r0->tail = r1->tail; + + return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0); +} +#endif diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h index 190ca3a9b9e1..c77359f89285 100644 --- a/ggml/src/ggml-hexagon/htp/dma-queue.h +++ b/ggml/src/ggml-hexagon/htp/dma-queue.h @@ -3,8 +3,10 @@ #include <HAP_farf.h> #include <hexagon_types.h> +#include <assert.h> #include <stdbool.h> #include <stdint.h> +#include <string.h> #include "hex-utils.h" #include "hex-profile.h" @@ -24,8 +26,8 @@ typedef struct dma_descriptor_1d_s { uint32_t src_bypass:1; uint32_t order:1; uint32_t done:1; - void * src; - void * dst; + uint32_t src; + uint32_t dst; } dma_descriptor_1d; #if __HVX_ARCH__ < 75 @@ -40,8 +42,8 @@ typedef struct dma_descriptor_2d_s { uint32_t src_bypass:1; uint32_t order:1; uint32_t done:1; - void * src; - void * dst; + uint32_t src; + uint32_t dst; uint32_t desc_type:8; uint32_t reserved1:24; uint32_t row_size:16; @@ -64,10 +66,18 @@ typedef struct dma_descriptor_2d_s { uint32_t src_bypass:1; uint32_t order:1; uint32_t done:1; - void * src; - void * dst; + uint32_t src; + uint32_t dst; uint32_t desc_type:8; +#if __HVX_ARCH__ > 79 + uint32_t src_upper:8; + uint32_t dst_upper:8; + uint32_t allocation:2; + uint32_t reserved0:2; + uint32_t transform:4; +#else uint32_t reserved0:24; +#endif uint32_t row_size:24; uint32_t nrows_lo:8; uint32_t nrows_hi:8; @@ -78,45 +88,63 @@ typedef struct dma_descriptor_2d_s { #endif +#if __HVX_ARCH__ > 79 +typedef uint64_t dma_addr_t; +#else +typedef uint32_t dma_addr_t; +#endif + typedef struct { - void *dst; - const void *src; -} dma_ptr; + dma_addr_t dst; + dma_addr_t src; +} dma_data; + +// Hardware descriptor field limits +#define DMA_MAX_NROWS 0xFFFFu // 16-bit HW descriptor limit (65535) +#define DMA_MAX_SIZE_16B 0xFFFFu // 16-bit HW descriptor limit for row_size (65535) +#define DMA_MAX_STRIDE_16B 0xFFFFu // 16-bit HW descriptor limit for strides (65535) +#define DMA_MAX_SIZE_24B 0x00FFFFFFu // 24-bit HW descriptor limit for row_size / 1D size (16MB - 1) +#define DMA_MAX_STRIDE_24B 0x00FFFFFFu // 24-bit HW descriptor limit for strides (16MB - 1) +#define DMA_SAFE_CHUNK_SIZE 0x00F00000u // ~15MB safe contiguous chunk size + +#define DMA_FALLBACK_CAPACITY 16u // descriptors in secondary fallback ring typedef struct dma_ring_s dma_ring; struct dma_ring_s { dma_descriptor_2d * desc; // descriptor pointers dma_descriptor_2d * tail; // tail pointer - dma_ptr * dptr; // dst/src pointers + dma_data * data; // dst/src data uint32_t push_idx; uint32_t pop_idx; uint32_t capacity; uint32_t idx_mask; struct htp_thread_trace * trace; - uintptr_t vtcm_base; - uintptr_t vtcm_end; }; typedef struct dma_queue_s dma_queue; typedef dma_queue * dma_queue_t; struct dma_queue_s { - dma_ring * ring; // Points to the descriptor ring state - uint8_t nocache; // Queue-specific bypass flag + dma_ring * ring0; // Main descriptor ring state + dma_ring * ring1; // Secondary fallback descriptor ring state bool alias; // When set, dma_queue_delete will not free the ring }; - - size_t dma_queue_sizeof(size_t capacity); size_t dma_queue_alignof(void); -dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace); +dma_queue_t dma_queue_init(void * ptr, size_t capacity, struct htp_thread_trace * trace); void dma_queue_free(dma_queue_t q); size_t dma_queue_alias_sizeof(void); -dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache); +dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q); void dma_queue_alias_free(dma_queue_t q); +bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows); +bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total); +#if __HVX_ARCH__ < 75 +bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows); +#endif + // TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead // but those do not seem to always compiler properly. static inline void dmstart(void * next) { @@ -141,36 +169,37 @@ static inline unsigned int dmwait(void) { return ret; } -static inline dma_ptr dma_make_ptr(void *dst, const void *src) +static inline dma_data dma_make_data_impl(dma_addr_t dst, dma_addr_t src) { - dma_ptr p = { dst, src }; - return p; + dma_data d = { dst, src }; + return d; } -static inline bool dma_is_vtcm(const dma_queue * q, const void * ptr) { - return (uintptr_t) ptr >= q->ring->vtcm_base && (uintptr_t) ptr < q->ring->vtcm_end; -} +#define dma_make_data(dst, src) dma_make_data_impl((dma_addr_t) (dst), (dma_addr_t) (src)) + +static inline bool dma_ring_push_single_1d(dma_ring * r, dma_data ddata, size_t size) { +#if __HVX_ARCH__ > 79 + assert(!((ddata.src | ddata.dst) >> 32) || size == 0); +#endif -static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t size) { - dma_ring * r = q->ring; if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) { return false; } dma_descriptor_1d * desc = (dma_descriptor_1d *) &r->desc[r->push_idx]; - desc->src = (void *) dptr.src; - desc->dst = (void *) dptr.dst; + desc->src = (uint32_t) ddata.src; + desc->dst = (uint32_t) ddata.dst; desc->size = size; - r->dptr[r->push_idx] = dptr; + r->data[r->push_idx] = ddata; htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx); if (size) { desc->next = NULL; desc->desc_size = 0; // 1D mode - desc->src_bypass = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache; - desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache; + desc->src_bypass = 1; + desc->dst_bypass = 1; desc->order = 0; desc->done = 0; @@ -185,8 +214,17 @@ static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t return true; } -static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - dma_ring * r = q->ring; +static inline bool dma_ring_push_single_2d(dma_ring * r, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { +#if __HVX_ARCH__ > 79 + const uint32_t src_hi = (uint32_t) (ddata.src >> 32); + const uint32_t dst_hi = (uint32_t) (ddata.dst >> 32); + const bool is_ext = (src_hi | dst_hi) != 0; + + if (is_ext && ((ddata.src >> 40) || (ddata.dst >> 40))) { + return false; + } +#endif + if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) { return false; } @@ -194,34 +232,44 @@ static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dma_descriptor_2d * desc = &r->desc[r->push_idx]; desc->next = NULL; - desc->reserved0 = 0; desc->reserved1 = 0; desc->desc_size = 1; // 2d mode - desc->src_bypass = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache; - desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache; + desc->src_bypass = 1; + desc->dst_bypass = 1; desc->src_comp = 0; desc->dst_comp = 0; desc->order = 0; desc->done = 0; desc->src_stride = src_stride; desc->dst_stride = dst_stride; - desc->src = (void *) dptr.src; - desc->dst = (void *) dptr.dst; + desc->src = (uint32_t) ddata.src; + desc->dst = (uint32_t) ddata.dst; desc->row_size = row_size; #if __HVX_ARCH__ < 75 + desc->reserved0 = 0; desc->desc_type = 0; // 2d (16-bit) mode desc->nrows = nrows; desc->src_offset = 0; desc->dst_offset = 0; #else +#if __HVX_ARCH__ > 79 + desc->src_upper = src_hi; + desc->dst_upper = dst_hi; + desc->allocation = 0; + desc->reserved0 = 0; + desc->transform = 0; + desc->desc_type = is_ext ? 10 : 9; // 2d 40-bit or 24-bit mode +#else + desc->reserved0 = 0; desc->desc_type = 9; // 2d (24-bit) mode +#endif desc->nrows_lo = (nrows & 0xff); desc->nrows_hi = (nrows >> 8); desc->offset = 0; #endif - r->dptr[r->push_idx] = dptr; + r->data[r->push_idx] = ddata; htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx); @@ -236,21 +284,20 @@ static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t return true; } -static inline dma_ptr dma_queue_pop(dma_queue * q) { - dma_ring * r = q->ring; - dma_ptr dptr = { NULL }; +static inline dma_data dma_ring_pop(dma_ring * r) { + dma_data ddata = { 0 }; if (r->push_idx == r->pop_idx) { - return dptr; + return ddata; } - dptr = r->dptr[r->pop_idx]; + ddata = r->data[r->pop_idx]; volatile dma_descriptor_2d * desc = &r->desc[r->pop_idx]; // Wait for desc to complete if (!desc->done) { - // FARF(ALWAYS, "dma-poll: idx %u dst %p src %p", r->pop_idx, dptr.dst, dptr.src); + // FARF(ALWAYS, "dma-poll: idx %u dst %p src %p", r->pop_idx, ddata.dst, ddata.src); while (!desc->done) { dmpoll(); } @@ -259,112 +306,138 @@ static inline dma_ptr dma_queue_pop(dma_queue * q) { htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx); r->pop_idx = (r->pop_idx + 1) & r->idx_mask; - return dptr; + return ddata; } -static inline dma_ptr dma_queue_pop_nowait(dma_queue * q) { - dma_ring * r = q->ring; - dma_ptr dptr = { NULL }; +static inline dma_data dma_ring_pop_nowait(dma_ring * r) { + dma_data ddata = { 0 }; if (r->push_idx == r->pop_idx) { - return dptr; + return ddata; } - dptr = r->dptr[r->pop_idx]; + ddata = r->data[r->pop_idx]; htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx); r->pop_idx = (r->pop_idx + 1) & r->idx_mask; - return dptr; + return ddata; +} + +static inline bool dma_ring_empty(dma_ring * r) { + return r->push_idx == r->pop_idx; +} + +static inline void dma_ring_flush(dma_ring * r) { + while (!dma_ring_empty(r)) { + dma_ring_pop(r); + } +} + +static inline uint32_t dma_ring_depth(dma_ring * r) { + return (r->push_idx - r->pop_idx) & r->idx_mask; +} + +static inline uint32_t dma_ring_capacity(dma_ring * r) { + return r->capacity; +} + +static inline bool dma_queue_push_single_1d(dma_queue * q, dma_data ddata, size_t size) { + return dma_ring_push_single_1d(q->ring0, ddata, size); +} + +static inline bool dma_queue_push_single_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + return dma_ring_push_single_2d(q->ring0, ddata, dst_stride, src_stride, row_size, nrows); +} + +static inline dma_data dma_queue_pop(dma_queue * q) { + return dma_ring_pop(q->ring0); +} + +static inline dma_data dma_queue_pop_nowait(dma_queue * q) { + return dma_ring_pop_nowait(q->ring0); } static inline bool dma_queue_empty(dma_queue * q) { - return q->ring->push_idx == q->ring->pop_idx; + return dma_ring_empty(q->ring0); } static inline void dma_queue_flush(dma_queue * q) { - while (dma_queue_pop(q).dst != NULL) ; + dma_ring_flush(q->ring0); } static inline uint32_t dma_queue_depth(dma_queue * q) { - return (q->ring->push_idx - q->ring->pop_idx) & q->ring->idx_mask; + return dma_ring_depth(q->ring0); } static inline uint32_t dma_queue_capacity(dma_queue * q) { - return q->ring->capacity; + return dma_ring_capacity(q->ring0); } #if __HVX_ARCH__ < 75 -// Overflow-safe DMA push: all 2d descriptor fields (row_size, nrows, src_stride, dst_stride) are 16-bit, max 65535. -// This version transparently handles values that exceed the 16-bit limit and submits chained DMA transtions. - -#define DMA_MAX_FIELD_VAL 65535u - -static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { +static inline bool dma_queue_push(dma_queue *q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { // Fast path: everything fits in 16 bits if (nrows == 0 || __builtin_expect( - row_size <= DMA_MAX_FIELD_VAL && - nrows <= DMA_MAX_FIELD_VAL && - src_stride <= DMA_MAX_FIELD_VAL && - dst_stride <= DMA_MAX_FIELD_VAL, 1)) { - return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); + nrows <= DMA_MAX_NROWS && + row_size <= DMA_MAX_SIZE_16B && + src_stride <= DMA_MAX_STRIDE_16B && + dst_stride <= DMA_MAX_STRIDE_16B, 1)) { + return dma_ring_push_single_2d(q->ring0, ddata, dst_stride, src_stride, row_size, nrows); } - // Contiguous block - // Use 1d DMA mode which supports sizes up to 24-bits (16MB) + // Contiguous block: 1D DMA mode supports up to 24-bit size (16MB) if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) { size_t total = row_size * nrows; - return dma_queue_push_single_1d(q, dptr, total); + if (total <= DMA_MAX_SIZE_24B) { + return dma_ring_push_single_1d(q->ring0, ddata, total); + } + return dma_queue_push_fallback_contig(q, ddata, total); } - // Stride overflow - fall back to row-by-row. - { - const uint8_t *src = (const uint8_t *) dptr.src; - uint8_t *dst = (uint8_t *) dptr.dst; - size_t r = 0; - while (r + 1 < nrows) { - dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride); - if (!dma_queue_push_single_1d(q, p, row_size)) { - dma_queue_flush(q); - } else { - r++; - } - } - dma_queue_flush(q); - dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride); - return dma_queue_push_single_1d(q, p, row_size); + // Row count overflow with 16-bit strides: chunk 2D descriptors via fallback ring + if (row_size <= DMA_MAX_SIZE_16B && src_stride <= DMA_MAX_STRIDE_16B && dst_stride <= DMA_MAX_STRIDE_16B) { + return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows); } + + // Stride or row_size overflow: row-by-row 1D via fallback ring + return dma_queue_push_fallback_1d(q, ddata, dst_stride, src_stride, row_size, nrows); } #else // HVX_ARCH >= 75 -static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - // On v75 and up we always use 2d 24-bit mode - return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); -} +static inline bool dma_queue_push(dma_queue *q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + if (nrows == 0 || __builtin_expect( + nrows <= DMA_MAX_NROWS && + row_size <= DMA_MAX_SIZE_24B && + src_stride <= DMA_MAX_STRIDE_24B && + dst_stride <= DMA_MAX_STRIDE_24B, 1)) { + return dma_ring_push_single_2d(q->ring0, ddata, dst_stride, src_stride, row_size, nrows); + } -#endif + // Contiguous block exceeding 24 bits + if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) { + size_t total = row_size * nrows; + return dma_queue_push_fallback_contig(q, ddata, total); + } -static inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { - return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows); + return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows); } -static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { - return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows); -} +#endif #define DMA_CACHE_MAX_SIZE 256U +// Fully assoc LRU cache typedef struct { uint8_t *base; uint32_t line_size; uint32_t capacity; - uint32_t src[DMA_CACHE_MAX_SIZE]; + dma_addr_t src[DMA_CACHE_MAX_SIZE]; uint16_t age[DMA_CACHE_MAX_SIZE]; -} dma_cache; +} dma_cache_fa; -static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity) +static inline void dma_cache_fa_init(dma_cache_fa *c, uint8_t *base, uint32_t line_size, uint32_t capacity) { c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity; c->base = base; @@ -376,14 +449,14 @@ static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_siz } } -static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * src, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows) +static inline bool dma_cache_fa_push(dma_queue *q, dma_cache_fa *c, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows) { uint32_t o_idx = 0; uint16_t o_age = 0; uint8_t * dst = 0; for (unsigned i=0; i < c->capacity; i++) { - if (c->src[i] == (uint32_t) src) { + if (c->src[i] == src_addr) { c->age[i] = 0; dst = c->base + (i * c->line_size); nrows = 0; // dummy dma } else { @@ -393,12 +466,46 @@ static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * sr } if (!dst) { c->age[o_idx] = 0; - c->src[o_idx] = (uint32_t) src; + c->src[o_idx] = src_addr; dst = c->base + o_idx * c->line_size; // normal nrows dma - return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows); + return dma_queue_push(q, dma_make_data(dst, src_addr), dst_stride, src_stride, row_size, nrows); + } + + return dma_queue_push_single_1d(q, dma_make_data(dst, src_addr), 0); +} + +// Direct mapped cache +typedef struct { + uint8_t *base; + uint32_t line_size; + uint32_t capacity; + uint32_t idx_mask; + dma_addr_t src[DMA_CACHE_MAX_SIZE]; +} dma_cache_dm; + +static inline void dma_cache_dm_init(dma_cache_dm *c, uint8_t *base, uint32_t line_size, uint32_t capacity) +{ + c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity; + c->idx_mask = c->capacity - 1; + c->base = base; + c->line_size = line_size; + + for (unsigned i=0; i < c->capacity; i++) { + c->src[i] = 0; + } +} + +static inline bool dma_cache_dm_push(dma_queue *q, dma_cache_dm *c, uint32_t slot, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows) +{ + const uint32_t i = slot & c->idx_mask; + uint8_t * dst = c->base + (i * c->line_size); + + if (c->src[i] == src_addr) { + return dma_queue_push_single_1d(q, dma_make_data(dst, src_addr), 0); // dummy dma } - return dma_queue_push_single_1d(q, dma_make_ptr(dst, src), 0); + c->src[i] = src_addr; + return dma_queue_push(q, dma_make_data(dst, src_addr), dst_stride, src_stride, row_size, nrows); } #ifdef __cplusplus diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.c b/ggml/src/ggml-hexagon/htp/fill-ops.c index 3ccfbe74ee45..212104a23bd4 100644 --- a/ggml/src/ggml-hexagon/htp/fill-ops.c +++ b/ggml/src/ggml-hexagon/htp/fill-ops.c @@ -3,10 +3,11 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include <HAP_farf.h> -#include <HAP_perf.h> - #include <string.h> +#include "hex-common.h" +#include "hex-profile.h" + #include "hvx-copy.h" #include "hvx-utils.h" @@ -14,28 +15,30 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" // ggml op_params layout for FILL: // op_params[0] (as float) - the scalar fill value -#define fill_preamble \ +#define fill_preamble \ const struct htp_tensor * dst = octx->dst; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ + \ const uint32_t nr = ne1 * ne2 * ne3; struct htp_fill_context { struct htp_ops_context * octx; uint32_t nrows_per_thread; uint32_t total_rows; // ne1 * ne2 * ne3 + uint32_t row_start; bool opt_path; HVX_Vector splat_vec; uint32_t elem_size; @@ -47,10 +50,15 @@ static void fill_thread(unsigned int nth, unsigned int ith, void * data) { fill_preamble; // Parallelise over the flat row index spanning ne1*ne2*ne3 - const uint32_t ir0 = fctx->nrows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->total_rows); + const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows); - uint64_t t1 = HAP_perf_get_qtimer_count(); + if (ir0 >= ir1) { + return; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); if (fctx->opt_path) { // Opt path: tensor is fully contiguous, treat as flat array @@ -69,9 +77,8 @@ static void fill_thread(unsigned int nth, unsigned int ith, void * data) { } } - uint64_t t2 = HAP_perf_get_qtimer_count(); - FARF(HIGH, "fill %u/%u: rows %u:%u usec %u\n", - ith, nth, ir0, ir1, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1); + FARF(HIGH, "fill %u/%u: rows %u:%u\n", ith, nth, ir0, ir1); } int op_fill(struct htp_ops_context * octx) { @@ -81,12 +88,27 @@ int op_fill(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + if (htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + uint32_t row_start = 0; + uint32_t nrows = nr; + + if (octx->ctx->mdev.count > 1) { + const uint32_t row_size = nb1; + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(nr, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { return HTP_STATUS_OK; } // nr = ne1*ne2*ne3 (flat row count across all outer dims); parallelise over it. - const uint32_t n_threads = MIN(nr, octx->n_threads); + const uint32_t n_threads = octx->n_threads; // Optimize if fully contiguous: skip stride arithmetic, treat as flat array const bool opt_path = (nb2 == nb1 * ne1) && (nb3 == nb2 * ne2); @@ -99,8 +121,9 @@ int op_fill(struct htp_ops_context * octx) { struct htp_fill_context fctx = { .octx = octx, - .nrows_per_thread = (nr + n_threads - 1) / n_threads, - .total_rows = nr, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_rows = nrows, + .row_start = row_start, .opt_path = opt_path, }; @@ -117,7 +140,7 @@ int op_fill(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - worker_pool_run_func(octx->ctx->worker_pool, fill_thread, &fctx, n_threads); + work_queue_run(octx->ctx->work_queue, fill_thread, &fctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index c76b4d3a3ac6..f079f738920a 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -5,7 +5,6 @@ #include <assert.h> #include <HAP_compute_res.h> #include <HAP_farf.h> -#include <HAP_perf.h> #include <math.h> #include <stdbool.h> #include <stdatomic.h> @@ -13,7 +12,7 @@ #include <stdint.h> #include <string.h> -#include "hex-dma.h" +#include "dma-queue.h" #include "hex-fastdiv.h" #include "hex-profile.h" #include "hmx-queue.h" @@ -56,6 +55,7 @@ struct htp_fa_context { float scale; float max_bias; + bool has_softcap; __fp16 logit_softcap; uint32_t n_head_log2; @@ -75,6 +75,7 @@ struct htp_fa_context { uint32_t qrows; uint32_t qrows_per_thread; + uint32_t qrow_start; bool is_q_fp32; @@ -86,11 +87,10 @@ struct htp_fa_context { uint8_t * spad_v; uint8_t * spad_m; uint8_t * spad_a; + float * spad_sinks; const struct htp_tensor * k; const struct htp_tensor * v; - - uint64_t t_start; }; struct hmx_fa_context { @@ -104,12 +104,14 @@ struct hmx_fa_context { // Op parameters __fp16 scale; float max_bias; + bool has_softcap; __fp16 logit_softcap; uint32_t n_head_log2; float m0, m1; // Dimensions uint32_t DK, DV; + uint32_t DK_pad, DV_pad; // head_dim rounded up to 64 for HMX tiling uint32_t n_kv; // kv_len uint32_t n_kv_heads; // number of KV heads uint32_t n_heads; // number of Q heads @@ -150,6 +152,7 @@ struct hmx_fa_context { uint8_t * vtcm_hmx_scales_qk; // HMX output scales (qk_scale) __fp16 * vtcm_mask_buf; // VTCM mask buffer [Br * m_line], DMA'd per KV block __fp16 * vtcm_slopes; // ALiBi slopes [g_br] + float * vtcm_sinks; // Attention sinks size_t row_buf_stride; // HVX vectors per row buffer (Bc/64) size_t mask_buf_row_stride; // elements (__fp16) per row in mask buffer size_t q_tile_bytes; @@ -157,7 +160,7 @@ struct hmx_fa_context { size_t col_vec_bytes; size_t d_tile_bytes; bool mask_broadcast; // true when mask->ne[2] == 1 (head-independent, single 2D DMA) - dma_cache m_cache; + dma_cache_fa m_cache; }; static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) { @@ -206,16 +209,15 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint32_t nb3 = dst->nb[3]; // total rows in q - const uint32_t nr = factx->qrows; - const uint32_t dr = factx->qrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, nr); + const uint32_t dr = factx->qrows_per_thread; + const uint32_t ir0 = factx->qrow_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, factx->qrow_start + factx->qrows); if (ir0 >= ir1) return; struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - dma_queue * dma = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; const uint32_t DK = nek0; const uint32_t DV = nev0; @@ -231,10 +233,13 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * uint8_t * spad_m = factx->spad_m + (mask ? factx->size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * ith; uint8_t * spad_a = factx->spad_a + factx->size_vkq_acc * ith; - dma_cache m_cache; - dma_cache_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE); + dma_cache_dm m_cache; + dma_cache_dm_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE); + + const size_t size_vkq_acc_single = hex_round_up(DV * sizeof(float), 128); - for (uint32_t ir = ir0; ir < ir1; ++ir) { + uint32_t ir = ir0; + while (ir < ir1) { const uint32_t iq3 = fastdiv(ir, &factx->src0_div21); const uint32_t iq2 = fastdiv(ir - iq3*neq2*neq1, &factx->src0_div1); const uint32_t iq1 = (ir - iq3*neq2*neq1 - iq2 * neq1); @@ -245,131 +250,112 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint32_t iv3 = fastdiv(iq3, &factx->broadcast_rv3); const uint32_t iv2 = fastdiv(iq2, &factx->broadcast_rv2); - const __fp16 * mp_base = NULL; - if (mask) { - const uint32_t im2 = fastmodulo(iq2, mask->ne[2], &factx->src3_div2); - const uint32_t im3 = fastmodulo(iq3, mask->ne[3], &factx->src3_div3); - mp_base = (const __fp16 *) ((const uint8_t *) mask->data + iq1*mask->nb[1] + im2*mask->nb[2] + im3*mask->nb[3]); - } + uint32_t G_local = 1; + if (neq1 == 1 && (mask == NULL || mask->ne[2] == 1)) { + while (ir + G_local < ir1 && G_local < FA_HVX_G_MAX) { + const uint32_t next_ir = ir + G_local; + const uint32_t next_iq3 = fastdiv(next_ir, &factx->src0_div21); + const uint32_t next_iq2 = fastdiv(next_ir - next_iq3*neq2*neq1, &factx->src0_div1); + const uint32_t next_iq1 = (next_ir - next_iq3*neq2*neq1 - next_iq2 * neq1); - // Precalculate next row variables if there is a next row - bool has_next_ir = (ir + 1 < ir1); - uint32_t next_ik2 = 0, next_ik3 = 0, next_iv2 = 0, next_iv3 = 0; - const uint8_t * next_q_row_ptr = NULL; - const __fp16 * next_mp_base = NULL; + const uint32_t next_ik3 = fastdiv(next_iq3, &factx->broadcast_rk3); + const uint32_t next_ik2 = fastdiv(next_iq2, &factx->broadcast_rk2); - const uint8_t * next_k_src0 = NULL; - const uint8_t * next_v_src0 = NULL; - const uint8_t * next_m_src0 = NULL; - uint32_t next_block_size0 = 0; + const uint32_t next_iv3 = fastdiv(next_iq3, &factx->broadcast_rv3); + const uint32_t next_iv2 = fastdiv(next_iq2, &factx->broadcast_rv2); - const uint8_t * next_k_src1 = NULL; - const uint8_t * next_v_src1 = NULL; - const uint8_t * next_m_src1 = NULL; - uint32_t next_block_size1 = 0; + if (next_ik2 != ik2 || next_ik3 != ik3 || next_iv2 != iv2 || next_iv3 != iv3 || next_iq1 != iq1 || next_iq3 != iq3) { + break; + } + G_local++; + } + } - if (has_next_ir) { - const uint32_t next_ir = ir + 1; - const uint32_t next_iq3 = fastdiv(next_ir, &factx->src0_div21); - const uint32_t next_iq2 = fastdiv(next_ir - next_iq3*neq2*neq1, &factx->src0_div1); - const uint32_t next_iq1 = (next_ir - next_iq3*neq2*neq1 - next_iq2 * neq1); + uint32_t heads[FA_HVX_G_MAX]; + HVX_Vector slope_vecs[FA_HVX_G_MAX] __attribute__((aligned(128))); + HVX_Vector S_vec[FA_HVX_G_MAX] __attribute__((aligned(128))); + HVX_Vector M_vec[FA_HVX_G_MAX] __attribute__((aligned(128))); + uint8_t * q_ptrs[FA_HVX_G_MAX]; + float * vkq_ptrs[FA_HVX_G_MAX]; - next_ik3 = fastdiv(next_iq3, &factx->broadcast_rk3); - next_ik2 = fastdiv(next_iq2, &factx->broadcast_rk2); + for (uint32_t g = 0; g < G_local; ++g) { + const uint32_t r = ir + g; + const uint32_t r_iq3 = fastdiv(r, &factx->src0_div21); + const uint32_t r_iq2 = fastdiv(r - r_iq3*neq2*neq1, &factx->src0_div1); + const uint32_t r_iq1 = (r - r_iq3*neq2*neq1 - r_iq2 * neq1); - next_iv3 = fastdiv(next_iq3, &factx->broadcast_rv3); - next_iv2 = fastdiv(next_iq2, &factx->broadcast_rv2); + heads[g] = r_iq2; + const __fp16 slope = factx->slopes[r_iq2]; + slope_vecs[g] = hvx_vec_splat_f16(slope); - next_q_row_ptr = (const uint8_t *) q->data + (next_iq1*nbq1 + next_iq2*nbq2 + next_iq3*nbq3); + S_vec[g] = hvx_vec_splat_f32(0.0f); + M_vec[g] = hvx_vec_splat_f32(HTP_FA_M_INITIAL_VAL); - if (mask) { - const uint32_t next_im2 = fastmodulo(next_iq2, mask->ne[2], &factx->src3_div2); - const uint32_t next_im3 = fastmodulo(next_iq3, mask->ne[3], &factx->src3_div3); - next_mp_base = (const __fp16 *) ((const uint8_t *) mask->data + next_iq1*mask->nb[1] + next_im2*mask->nb[2] + next_im3*mask->nb[3]); - } + uint8_t * q_dst = spad_q + g * factx->size_q_row_padded; + q_ptrs[g] = q_dst; - // Precalculate next K/V block 0 source pointers - { - const uint32_t ic_start = 0; - next_block_size0 = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start); - next_k_src0 = (const uint8_t *) k->data + (ic_start*nbk1 + next_ik2*nbk2 + next_ik3*nbk3); - next_v_src0 = (const uint8_t *) v->data + (ic_start*nbv1 + next_iv2*nbv2 + next_iv3*nbv3); - if (mask) { - next_m_src0 = (const uint8_t *) (next_mp_base + ic_start); - } - } + float * vkq_dst = (float *)(spad_a + g * size_vkq_acc_single); + vkq_ptrs[g] = vkq_dst; + hvx_splat_f32_a((uint8_t *) vkq_dst, 0, DV); - // Precalculate next K/V block 1 source pointers (if n_blocks > 1) - if (factx->n_blocks > 1) { - const uint32_t ic_start = 1 * FLASH_ATTN_BLOCK_SIZE; - next_block_size1 = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start); - next_k_src1 = (const uint8_t *) k->data + (ic_start*nbk1 + next_ik2*nbk2 + next_ik3*nbk3); - next_v_src1 = (const uint8_t *) v->data + (ic_start*nbv1 + next_iv2*nbv2 + next_iv3*nbv3); - if (mask) { - next_m_src1 = (const uint8_t *) (next_mp_base + ic_start); - } - } + // Fetch Q row g + const dma_addr_t q_row_ptr = q->data + r_iq1*nbq1 + r_iq2*nbq2 + r_iq3*nbq3; + dma_queue_push(dma_q, dma_make_data(q_dst, q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1); } - if (ir == ir0) { - // Fetch Q row - const uint8_t * q_row_ptr = (const uint8_t *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3); - dma_queue_push(dma, dma_make_ptr(spad_q, q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1); + dma_addr_t mp_base = 0; + if (mask) { + const uint32_t im2 = fastmodulo(iq2, mask->ne[2], &factx->src3_div2); + const uint32_t im3 = fastmodulo(iq3, mask->ne[3], &factx->src3_div3); + mp_base = mask->data + iq1*mask->nb[1] + im2*mask->nb[2] + im3*mask->nb[3]; + } - // Prefetch first two blocks - for (uint32_t ib = 0; ib < MIN(factx->n_blocks, 2); ++ib) { - const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE; - const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start); + // Prefetch first two blocks + for (uint32_t ib = 0; ib < MIN(factx->n_blocks, 2); ++ib) { + const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE; + const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start); - // K - const uint8_t * k_src = (const uint8_t *) k->data + (ic_start*nbk1 + ik2*nbk2 + ik3*nbk3); - uint8_t * k_dst = spad_k + (ib % 2) * factx->size_k_block; - dma_queue_push(dma, dma_make_ptr(k_dst, k_src), factx->size_k_row_padded, nbk1, size_k_row, current_block_size); + // K + const dma_addr_t k_src = k->data + ic_start*nbk1 + ik2*nbk2 + ik3*nbk3; + uint8_t * k_dst = spad_k + (ib % 2) * factx->size_k_block; + dma_queue_push(dma_q, dma_make_data(k_dst, k_src), factx->size_k_row_padded, nbk1, size_k_row, current_block_size); - // V - const uint8_t * v_src = (const uint8_t *) v->data + (ic_start*nbv1 + iv2*nbv2 + iv3*nbv3); - uint8_t * v_dst = spad_v + (ib % 2) * factx->size_v_block; - dma_queue_push(dma, dma_make_ptr(v_dst, v_src), factx->size_v_row_padded, nbv1, size_v_row, current_block_size); + // V + const dma_addr_t v_src = v->data + ic_start*nbv1 + iv2*nbv2 + iv3*nbv3; + uint8_t * v_dst = spad_v + (ib % 2) * factx->size_v_block; + dma_queue_push(dma_q, dma_make_data(v_dst, v_src), factx->size_v_row_padded, nbv1, size_v_row, current_block_size); - // Mask - if (mask) { - const uint8_t * m_src = (const uint8_t *) (mp_base + ic_start); - // Mask is 1D contiguous for this row - dma_cache_push(dma, &m_cache, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1); - } + // Mask + if (mask) { + const dma_addr_t m_src = mp_base + ic_start * sizeof(__fp16); + dma_cache_dm_push(dma_q, &m_cache, ib, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1); } } - const uint32_t h = iq2; // head index - const __fp16 slope = factx->slopes[h]; - - HVX_Vector S_vec = hvx_vec_splat_f32(0.0f); - HVX_Vector M_vec = hvx_vec_splat_f32(HTP_FA_M_INITIAL_VAL); - - // Clear accumulator - hvx_splat_f32_a(spad_a, 0, DV); - float * VKQ32 = (float *) (spad_a + 0); - - uint8_t * q_ptr_vtcm = dma_queue_pop(dma).dst; - if (factx->is_q_fp32) { - hvx_copy_f16_f32_aa(q_ptr_vtcm, q_ptr_vtcm, DK); // inplace convert f32 to f16 + // Pop all Q rows + for (uint32_t g = 0; g < G_local; ++g) { + uint8_t * q_ptr_vtcm = (void *) dma_queue_pop(dma_q).dst; + if (factx->is_q_fp32) { + hvx_copy_f16_f32_aa(q_ptr_vtcm, q_ptr_vtcm, DK); + } } - const HVX_Vector slope_vec = hvx_vec_splat_f16(slope); const HVX_Vector v_neg_inf = Q6_Vh_vsplat_R(0xfbff); - const HVX_Vector v_cap = (factx->logit_softcap != 0.0f) ? hvx_vec_splat_f16(factx->logit_softcap) : Q6_V_vzero(); + const bool has_softcap = factx->has_softcap; + const HVX_Vector v_cap = has_softcap ? hvx_vec_splat_f16(factx->logit_softcap) : Q6_V_vzero(); const HVX_Vector vinf = Q6_Vh_vsplat_R(0xFC00); const HVX_Vector vmin = Q6_Vh_vsplat_R(0xFBFF); const HVX_Vector v_log2e = hvx_vec_splat_f16(EXP_LOG2E_F); const uint32_t stride_v2 = factx->size_v_row_padded * 2; + for (uint32_t ib = 0; ib < factx->n_blocks; ++ib) { const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE; const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start); // Wait for DMA - uint8_t * k_base = dma_queue_pop(dma).dst; // K - uint8_t * v_base = dma_queue_pop(dma).dst; // V - __fp16 * m_base = mask ? dma_queue_pop(dma).dst : NULL; // M + uint8_t * k_base = (void *) dma_queue_pop(dma_q).dst; // K + uint8_t * v_base = (void *) dma_queue_pop(dma_q).dst; // V + __fp16 * m_base = mask ? (__fp16 *) dma_queue_pop(dma_q).dst : NULL; // M if (factx->k->type == HTP_TYPE_Q8_0) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir); @@ -388,235 +374,222 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir); } - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_QK, ir); - - // Inner loop processing the block from VTCM - // 1. Compute scores (64 elements FP16) - HVX_Vector scores_f16 = Q6_V_vzero(); - if (current_block_size > 0) { - HVX_Vector scores0 = hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base, factx->size_k_row_padded, DK, factx->scale); - HVX_Vector scores1 = (current_block_size > 32) ? hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base + 32 * factx->size_k_row_padded, factx->size_k_row_padded, DK, factx->scale) : Q6_V_vzero(); - scores_f16 = hvx_vec_f32_to_f16(scores0, scores1); - } - - // 2. Softcap (in FP16) - if (factx->logit_softcap != 0.0f) { - scores_f16 = hvx_vec_tanh_f16(scores_f16); - scores_f16 = hvx_vec_mul_f16_f16(scores_f16, v_cap); - } + for (uint32_t g = 0; g < G_local; ++g) { + const uint32_t head_ir = ir + g; + uint8_t * q_ptr_vtcm = q_ptrs[g]; + float * VKQ32 = vkq_ptrs[g]; + const HVX_Vector slope_vec = slope_vecs[g]; - HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(current_block_size * sizeof(__fp16)); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_QK, head_ir); - // 3. Mask (in FP16) - if (mask) { - HVX_Vector m_vals_f16 = *(const HVX_UVector *) m_base; - HVX_VectorPred is_inf = Q6_Q_vcmp_eq_VhVh(m_vals_f16, vinf); - m_vals_f16 = Q6_V_vmux_QVV(is_inf, vmin, m_vals_f16); + HVX_Vector scores_f16 = Q6_V_vzero(); + if (current_block_size > 0) { + HVX_Vector scores0 = hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base, factx->size_k_row_padded, DK, factx->scale); + HVX_Vector scores1 = (current_block_size > 32) ? hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base + 32 * factx->size_k_row_padded, factx->size_k_row_padded, DK, factx->scale) : Q6_V_vzero(); + scores_f16 = hvx_vec_f32_to_f16(scores0, scores1); + } - HVX_Vector m_scaled = hvx_vec_mul_f16_f16(m_vals_f16, slope_vec); - scores_f16 = Q6_V_vmux_QVV(q_tail_keep, hvx_vec_add_f16_f16(scores_f16, m_scaled), v_neg_inf); - } else { - scores_f16 = Q6_V_vmux_QVV(q_tail_keep, scores_f16, v_neg_inf); - } + if (has_softcap) { + scores_f16 = hvx_vec_tanh_f16(scores_f16); + scores_f16 = hvx_vec_mul_f16_f16(scores_f16, v_cap); + } - // Compute block max in FP16 - HVX_Vector v_max_f16 = hvx_vec_reduce_max_f16(scores_f16); - HVX_Vector v_max = Q6_V_lo_W(hvx_vec_f16_to_f32(v_max_f16)); // splat block max in FP32 - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_QK, ir); + HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(current_block_size * sizeof(__fp16)); - if (ib + 1 == factx->n_blocks && has_next_ir) { - // Queue next row's Q row! - dma_queue_push(dma, dma_make_ptr(spad_q, next_q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1); + if (mask) { + HVX_Vector m_vals_f16 = *(const HVX_UVector *) m_base; + HVX_VectorPred is_inf = Q6_Q_vcmp_eq_VhVh(m_vals_f16, vinf); + m_vals_f16 = Q6_V_vmux_QVV(is_inf, vmin, m_vals_f16); - if (factx->n_blocks % 2 == 0) { - // Queue next row's block 0 (into buffer slot 0) - uint8_t * k_dst = spad_k + 0 * factx->size_k_block; - uint8_t * v_dst = spad_v + 0 * factx->size_v_block; + HVX_Vector m_scaled = hvx_vec_mul_f16_f16(m_vals_f16, slope_vec); + scores_f16 = Q6_V_vmux_QVV(q_tail_keep, hvx_vec_add_f16_f16(scores_f16, m_scaled), v_neg_inf); + } else { + scores_f16 = Q6_V_vmux_QVV(q_tail_keep, scores_f16, v_neg_inf); + } - // K (block 0 of next row) - dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src0), factx->size_k_row_padded, nbk1, size_k_row, next_block_size0); + HVX_Vector v_max_f16 = hvx_vec_reduce_max_f16(scores_f16); + HVX_Vector v_max = Q6_V_lo_W(hvx_vec_f16_to_f32(v_max_f16)); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_QK, head_ir); - // V (block 0 of next row) - dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src0), factx->size_v_row_padded, nbv1, size_v_row, next_block_size0); + // prefetch K for block ib + 2 after last head finished QK + if (g + 1 == G_local && ib + 2 < factx->n_blocks) { + const uint32_t next_ib = ib + 2; + const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE; + const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start); - // Mask (block 0 of next row) - if (mask) { - dma_cache_push(dma, &m_cache, next_m_src0, next_block_size0 * 2, next_block_size0 * 2, next_block_size0 * 2, 1); - } + const dma_addr_t k_src = k->data + next_ic_start*nbk1 + ik2*nbk2 + ik3*nbk3; + dma_queue_push(dma_q, dma_make_data(k_base, k_src), factx->size_k_row_padded, nbk1, size_k_row, next_block_size); } - } - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, ir); - { - // 4. Online Softmax Update - HVX_Vector M_new_vec = Q6_Vsf_vmax_VsfVsf(v_max, M_vec); - HVX_Vector diff_vec = HVX_OP_SUB_F32(M_vec, M_new_vec); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, head_ir); + { + HVX_Vector M_new_vec = Q6_Vsf_vmax_VsfVsf(v_max, M_vec[g]); + HVX_Vector diff_vec = HVX_OP_SUB_F32(M_vec[g], M_new_vec); - HVX_Vector diff_f16 = hvx_vec_f32_to_f16(diff_vec, diff_vec); - HVX_Vector diff_base2 = hvx_vec_mul_f16_f16(diff_f16, v_log2e); - HVX_Vector ms_f16 = hvx_vec_exp2_f16(diff_base2); - HVX_Vector ms_vec = Q6_V_lo_W(hvx_vec_f16_to_f32(ms_f16)); + HVX_Vector diff_f16 = hvx_vec_f32_to_f16(diff_vec, diff_vec); + HVX_Vector diff_base2 = hvx_vec_mul_f16_f16(diff_f16, v_log2e); + HVX_Vector ms_f16 = hvx_vec_exp2_f16(diff_base2); + HVX_Vector ms_vec = Q6_V_lo_W(hvx_vec_f16_to_f32(ms_f16)); - M_vec = M_new_vec; + M_vec[g] = M_new_vec; - hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec); + HVX_Vector v_m_vec_f16 = hvx_vec_f32_to_f16(M_vec[g], M_vec[g]); + HVX_Vector v_s_minus_m = Q6_Vqf16_vsub_VhfVhf(scores_f16, v_m_vec_f16); + HVX_Vector v_s_minus_m_base2 = hvx_vec_mul_f16_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m), v_log2e); - // Compute P = exp2((S - M) * log2(e)) in FP16 - HVX_Vector v_m_vec_f16 = hvx_vec_f32_to_f16(M_vec, M_vec); - HVX_Vector v_s_minus_m = Q6_Vqf16_vsub_VhfVhf(scores_f16, v_m_vec_f16); + HVX_Vector P = hvx_vec_exp2_f16(v_s_minus_m_base2); + P = Q6_V_vmux_QVV(q_tail_keep, P, Q6_V_vzero()); - HVX_Vector v_s_minus_m_base2 = hvx_vec_mul_f16_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m), v_log2e); + HVX_VectorPair P_pair = hvx_vec_f16_to_f32(P); + HVX_Vector P0 = Q6_V_lo_W(P_pair); + HVX_Vector P1 = Q6_V_hi_W(P_pair); + HVX_Vector p_sum_vec = hvx_vec_reduce_sum_f32(HVX_OP_ADD_F32(P0, P1)); - HVX_Vector P = hvx_vec_exp2_f16(v_s_minus_m_base2); - P = Q6_V_vmux_QVV(q_tail_keep, P, Q6_V_vzero()); + S_vec[g] = HVX_OP_ADD_F32(HVX_OP_MUL_F32(S_vec[g], ms_vec), p_sum_vec); - // Convert P to FP32 to update the running sum S_vec - HVX_VectorPair P_pair = hvx_vec_f16_to_f32(P); - HVX_Vector P0 = Q6_V_lo_W(P_pair); - HVX_Vector P1 = Q6_V_hi_W(P_pair); - HVX_Vector p_sum_vec = hvx_vec_reduce_sum_f32(HVX_OP_ADD_F32(P0, P1)); + const uint8_t * v_ptr = v_base; - S_vec = HVX_OP_ADD_F32(HVX_OP_MUL_F32(S_vec, ms_vec), p_sum_vec); + if (DV == 64) { + HVX_VectorPair vkq0 = *((const HVX_VectorPair *) VKQ32); + vkq0 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq0), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq0), ms_vec) + ); - // 5. Accumulate V (F16 * F16 -> F32 accumulator) - const uint8_t * v_ptr = v_base; + for (uint32_t j = 0; j < current_block_size; j += 2) { + HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); + const HVX_Vector * vx0 = (const HVX_Vector *) v_ptr; + if (j + 1 == current_block_size) { + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + break; + } - for (uint32_t j = 0; j < current_block_size; j += 2) { - if (j + 1 == current_block_size) { - HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); - hvx_mad_f32_f16_aa_vec(VKQ32, v_ptr, S0, DV); - break; - } + HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2)); + const HVX_Vector * vx1 = (const HVX_Vector *) (v_ptr + factx->size_v_row_padded); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx1[0]), S1); + v_ptr += stride_v2; + } - HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); - HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2)); + *((HVX_VectorPair *) VKQ32) = vkq0; + } else if (DV == 128) { + HVX_VectorPair vkq0 = ((const HVX_VectorPair *) VKQ32)[0]; + HVX_VectorPair vkq1 = ((const HVX_VectorPair *) VKQ32)[1]; + vkq0 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq0), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq0), ms_vec) + ); + vkq1 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq1), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq1), ms_vec) + ); + + for (uint32_t j = 0; j < current_block_size; j += 2) { + HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); + const HVX_Vector * vx0 = (const HVX_Vector *) v_ptr; + if (j + 1 == current_block_size) { + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx0[1]), S0); + break; + } - hvx_mad_f32_f16_aa_rx2_vec(VKQ32, v_ptr, v_ptr + factx->size_v_row_padded, S0, S1, DV); - v_ptr += stride_v2; - } - } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, ir); + HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2)); + const HVX_Vector * vx1 = (const HVX_Vector *) (v_ptr + factx->size_v_row_padded); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx1[0]), S1); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx0[1]), S0); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx1[1]), S1); + v_ptr += stride_v2; + } - // Issue DMA for next+1 block (if exists) - if (ib + 2 < factx->n_blocks) { - const uint32_t next_ib = ib + 2; - const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE; - const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start); + ((HVX_VectorPair *) VKQ32)[0] = vkq0; + ((HVX_VectorPair *) VKQ32)[1] = vkq1; + } else { + hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec); - // K - const uint8_t * k_src = (const uint8_t *) k->data + (next_ic_start*nbk1 + ik2*nbk2 + ik3*nbk3); - dma_queue_push(dma, dma_make_ptr(k_base, k_src), factx->size_k_row_padded, nbk1, size_k_row, next_block_size); + for (uint32_t j = 0; j < current_block_size; j += 2) { + if (j + 1 == current_block_size) { + HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); + hvx_mad_f32_f16_aa_vec(VKQ32, v_ptr, S0, DV); + break; + } - // V - const uint8_t * v_src = (const uint8_t *) v->data + (next_ic_start*nbv1 + iv2*nbv2 + iv3*nbv3); - dma_queue_push(dma, dma_make_ptr(v_base, v_src), factx->size_v_row_padded, nbv1, size_v_row, next_block_size); + HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); + HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2)); - // Mask - if (mask) { - const uint8_t * m_src = (const uint8_t *) (mp_base + next_ic_start); - dma_cache_push(dma, &m_cache, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1); + hvx_mad_f32_f16_aa_rx2_vec(VKQ32, v_ptr, v_ptr + factx->size_v_row_padded, S0, S1, DV); + v_ptr += stride_v2; + } + } } - } - } - - if (has_next_ir) { - if (factx->n_blocks % 2 == 0) { - // Queue next row's block 1 (into buffer slot 1, if n_blocks > 1) - if (factx->n_blocks > 1) { - uint8_t * k_dst = spad_k + 1 * factx->size_k_block; - uint8_t * v_dst = spad_v + 1 * factx->size_v_block; + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, head_ir); - // K (block 1 of next row) - dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src1), factx->size_k_row_padded, nbk1, size_k_row, next_block_size1); + // prefetch V and mask for block ib + 2 after last head finished V accumulation + if (g + 1 == G_local && ib + 2 < factx->n_blocks) { + const uint32_t next_ib = ib + 2; + const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE; + const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start); - // V (block 1 of next row) - dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src1), factx->size_v_row_padded, nbv1, size_v_row, next_block_size1); + // V + const dma_addr_t v_src = v->data + next_ic_start*nbv1 + iv2*nbv2 + iv3*nbv3; + dma_queue_push(dma_q, dma_make_data(v_base, v_src), factx->size_v_row_padded, nbv1, size_v_row, next_block_size); - // Mask (block 1 of next row) + // Mask if (mask) { - dma_cache_push(dma, &m_cache, next_m_src1, next_block_size1 * 2, next_block_size1 * 2, next_block_size1 * 2, 1); + const dma_addr_t m_src = mp_base + next_ic_start * sizeof(__fp16); + dma_cache_dm_push(dma_q, &m_cache, next_ib, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1); } } - } else { - // Queue next row's block 0 (into buffer slot 0) - { - uint8_t * k_dst = spad_k + 0 * factx->size_k_block; - uint8_t * v_dst = spad_v + 0 * factx->size_v_block; + } // end for g + } // end for ib - // K (block 0 of next row) - dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src0), factx->size_k_row_padded, nbk1, size_k_row, next_block_size0); + for (uint32_t g = 0; g < G_local; ++g) { + const uint32_t head_ir = ir + g; + const uint32_t h = heads[g]; + float * VKQ32 = vkq_ptrs[g]; - // V (block 0 of next row) - dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src0), factx->size_v_row_padded, nbv1, size_v_row, next_block_size0); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, head_ir); - // Mask (block 0 of next row) - if (mask) { - dma_cache_push(dma, &m_cache, next_m_src0, next_block_size0 * 2, next_block_size0 * 2, next_block_size0 * 2, 1); - } - } + float M = hvx_vec_get_f32(M_vec[g]); + float S = hvx_vec_get_f32(S_vec[g]); - // Queue next row's block 1 (into buffer slot 1, if n_blocks > 1) - if (factx->n_blocks > 1) { - uint8_t * k_dst = spad_k + 1 * factx->size_k_block; - uint8_t * v_dst = spad_v + 1 * factx->size_v_block; + if (sinks) { + const float s = factx->spad_sinks[h]; - // K (block 1 of next row) - dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src1), factx->size_k_row_padded, nbk1, size_k_row, next_block_size1); + float vs = 1.0f; - // V (block 1 of next row) - dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src1), factx->size_v_row_padded, nbv1, size_v_row, next_block_size1); + if (s > M) { + HVX_Vector diff_vec = hvx_vec_splat_f32(M - s); + HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec); + hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec); - // Mask (block 1 of next row) - if (mask) { - dma_cache_push(dma, &m_cache, next_m_src1, next_block_size1 * 2, next_block_size1 * 2, next_block_size1 * 2, 1); - } + float ms = hvx_vec_get_f32(ms_vec); + S = S * ms + vs; + } else { + HVX_Vector diff_vec = hvx_vec_splat_f32(s - M); + vs = hvx_vec_get_f32(hvx_vec_exp_f32(diff_vec)); + S += vs; } } - } - - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, ir); - // sinks - float M = hvx_vec_get_f32(M_vec); - float S = hvx_vec_get_f32(S_vec); - if (sinks) { - const float s = ((float *)((char *) sinks->data))[h]; + const float S_inv = S == 0.0f ? 0.0f : 1.0f/S; + hvx_scale_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, S_inv); - float vs = 1.0f; + const uint32_t r_iq3 = fastdiv(head_ir, &factx->src0_div21); + const uint32_t r_iq2 = fastdiv(head_ir - r_iq3*neq2*neq1, &factx->src0_div1); + const uint32_t r_iq1 = (head_ir - r_iq3*neq2*neq1 - r_iq2 * neq1); - if (s > M) { - HVX_Vector diff_vec = hvx_vec_splat_f32(M - s); - HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec); - hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec); + uint8_t * dst_ptr = (uint8_t *) dst->data + r_iq2 * dst->nb[1] + r_iq1 * dst->nb[2] + r_iq3 * dst->nb[3]; - float ms = hvx_vec_get_f32(ms_vec); - S = S * ms + vs; - } else { - HVX_Vector diff_vec = hvx_vec_splat_f32(s - M); - vs = hvx_vec_get_f32(hvx_vec_exp_f32(diff_vec)); - S += vs; + if (dst->type == HTP_TYPE_F32) { + hvx_copy_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV); + } else if (dst->type == HTP_TYPE_F16) { + hvx_copy_f16_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, head_ir); } - const float S_inv = S == 0.0f ? 0.0f : 1.0f/S; - hvx_scale_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, S_inv); - - // Store result - // dst indices - const uint32_t i1 = iq1; - const uint32_t i2 = iq2; - const uint32_t i3 = iq3; - - // dst is permuted: [DV, n_heads, n_tokens, n_seq] - // head stride is nb[1], token stride is nb[2], batch stride is nb[3] - uint8_t * dst_ptr = (uint8_t *) dst->data + i2 * dst->nb[1] + i1 * dst->nb[2] + i3 * dst->nb[3]; - - if (dst->type == HTP_TYPE_F32) { - hvx_copy_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV); - } else if (dst->type == HTP_TYPE_F16) { - hvx_copy_f16_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV); - } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, ir); + ir += G_local; } } @@ -655,7 +628,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) hvx_dequantize_row_q8_0_f16(row_k, row_k, factx->DK); } } - hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles[args->buf_idx], (const __fp16 *) args->curr_k, total_rows, factx->DK, + hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles[args->buf_idx], (const __fp16 *) args->curr_k, total_rows, factx->DK_pad, args->src_stride, start, end); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); } @@ -709,7 +682,7 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) hvx_dequantize_row_q8_0_f16(row_v, row_v, factx->DV); } } - hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV, + hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV_pad, args->src_stride, (uint32_t) args->n_col_tiles, start, end); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); } @@ -783,7 +756,7 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { const size_t m_end = hex_smin(m_start + m_bytes_per_t, col_vec_bytes); if (factx->sinks) { - const float * sinks_data = (const float *) (uintptr_t) factx->sinks->data; + const float * sinks_data = factx->vtcm_sinks; float * m_vec = (float *) factx->vtcm_m_vec; const size_t r_start = l_start / sizeof(float); const size_t r_end = l_end / sizeof(float); @@ -835,17 +808,22 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { const uint32_t kv_head = args->kv_head; const uint32_t ib3 = args->ib3; - assert(factx->DK == factx->DV); - const bool use_q_dma = (factx->vtcm_q_dma != NULL); __fp16 * q_tiles = factx->vtcm_q_tiles; + const size_t DK_pad = factx->DK_pad; if (use_q_dma) { const size_t g_rows_end = hex_smin(end, n_rows_g); const uint32_t d_limit = factx->is_q_fp32 ? DK / 32 : DK / 64; uint8_t * q_flat = (uint8_t *) factx->vtcm_q_dma; - if (factx->is_q_fp32) { + if (DK_pad != DK) { + if (factx->is_q_fp32) { + hmx_fa_q_prep_fp32_pad(q_tiles, q_flat, start, end, g_rows_end, DK, DK_pad, G, args->n_rows_q, &factx->div_G, args->q_transposed); + } else { + hmx_fa_q_prep_fp16_pad(q_tiles, q_flat, start, end, g_rows_end, DK, DK_pad, G, args->n_rows_q, &factx->div_G, args->q_transposed); + } + } else if (factx->is_q_fp32) { switch (d_limit) { case 2: hmx_fa_q_prep_fp32_d2(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break; case 4: hmx_fa_q_prep_fp32_d4(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break; @@ -861,7 +839,7 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { } else { // Fallback: direct-from-DDR/L2 path hmx_fa_q_prep_fallback(q_tiles, q->data, q->nb[1], q->nb[2], q->nb[3], - q_start, kv_head, ib3, start, end, n_rows_g, G, DK, factx->is_q_fp32, &factx->div_G); + q_start, kv_head, ib3, start, end, n_rows_g, G, DK, DK_pad, factx->is_q_fp32, &factx->div_G); } } @@ -955,6 +933,8 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { const uint32_t kv_head = args->kv_head; const uint32_t ib3 = args->ib3; + const size_t DV_pad = factx->DV_pad; + size_t q_idx = fastdiv(start, &factx->div_G); size_t h_idx = fastmodulo(start, G, &factx->div_G); @@ -964,7 +944,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { size_t r0 = r / HMX_FP16_TILE_N_ROWS; size_t r1 = r % HMX_FP16_TILE_N_ROWS; - const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV; + const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV_pad; for (uint32_t d = 0; d < DV / 32; ++d) { const HVX_Vector * in_tile = (const HVX_Vector *) (tile_row_base + d * HMX_FP16_TILE_N_ELMS); @@ -975,6 +955,16 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { *(HVX_UVector *) (out + d * 32) = Q6_V_hi_W(vp); } } + // Ragged tail: DV not a multiple of 32 (e.g. 72 -> last 8 lanes). Partial vector-write + // for the remaining (DV % 32) floats. + const uint32_t d_tail = DV / 32; + const uint32_t rem = DV - d_tail * 32; + if (rem) { + const HVX_Vector * in_tile = (const HVX_Vector *) (tile_row_base + d_tail * HMX_FP16_TILE_N_ELMS); + HVX_VectorPair vp = hvx_vec_f16_to_f32_shuff(in_tile[r1 / 2]); + HVX_Vector vd = (r1 % 2 == 0) ? Q6_V_lo_W(vp) : Q6_V_hi_W(vp); + hvx_vec_store_u((void *) (out + d_tail * 32), rem * sizeof(float), vd); + } h_idx++; if (h_idx == G) { @@ -1009,6 +999,9 @@ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { const uint32_t kv_head = args->kv_head; const uint32_t ib3 = args->ib3; + // O-tiles use the padded head dim (DV_pad); dst holds the real DV lanes. + const size_t DV_pad = factx->DV_pad; + size_t q_idx = fastdiv(start, &factx->div_G); size_t h_idx = fastmodulo(start, G, &factx->div_G); @@ -1018,7 +1011,7 @@ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { size_t r0 = r / HMX_FP16_TILE_N_ROWS; size_t r1 = r % HMX_FP16_TILE_N_ROWS; - const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV; + const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV_pad; for (uint32_t d = 0; d < DV / 64; ++d) { const __fp16 * in_dtile = tile_row_base + d * HMX_FP16_TILE_N_ELMS * 2; @@ -1031,6 +1024,17 @@ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { *(HVX_UVector *) (out + d * 64) = Q6_V_hi_W(vp); } } + // Ragged tail when DV is not a multiple of 64. + const uint32_t d_tail = DV / 64; + const uint32_t rem = DV - d_tail * 64; + if (rem) { + const __fp16 * in_dtile = tile_row_base + d_tail * HMX_FP16_TILE_N_ELMS * 2; + const HVX_Vector * pv_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2; + const HVX_Vector * pv_in1 = pv_in0 + 16; + HVX_VectorPair vp = Q6_W_vdeal_VVR(*pv_in1, *pv_in0, -2); + HVX_Vector vd = (r1 % 2 == 0) ? Q6_V_lo_W(vp) : Q6_V_hi_W(vp); + hvx_vec_store_u((void *) (out + d_tail * 64), rem * sizeof(__fp16), vd); + } h_idx++; if (h_idx == G) { @@ -1523,7 +1527,7 @@ static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) { const bool mask_broadcast = factx->mask_broadcast; const bool is_g1 = (args->G == 1); const bool has_alibi = args->has_alibi; - const bool has_softcap = (factx->logit_softcap != 0.0f); + const bool has_softcap = factx->has_softcap; fa_softmax_impl(n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap); } @@ -1558,9 +1562,9 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64); worker_callback_t softmax_fn = fa_softmax_thread; - if (sargs->mask == NULL && factx->logit_softcap == 0.0f && !sargs->has_alibi) { + if (sargs->mask == NULL && !factx->has_softcap && !sargs->has_alibi) { softmax_fn = fa_softmax_thread_nomask; - } else if (sargs->mask != NULL && factx->mask_broadcast && factx->logit_softcap == 0.0f && !sargs->has_alibi) { + } else if (sargs->mask != NULL && factx->mask_broadcast && !factx->has_softcap && !sargs->has_alibi) { if (sargs->G == 1) { softmax_fn = fa_softmax_thread_mask_broadcast_g1; } else { @@ -1750,7 +1754,7 @@ static __attribute__((noinline)) void fa_compute_slopes( } static void fa_push_mask_dma_gqa( - dma_queue * dma, + dma_queue * dma_q, const struct htp_tensor * mask, uint32_t q_start, uint32_t im3, @@ -1765,36 +1769,36 @@ static void fa_push_mask_dma_gqa( for (uint32_t g = 0; g < G; ++g) { const uint32_t h_idx = kv_head * G + g; const uint32_t im2 = fastmodulo(h_idx, mask->ne[2], &factx->src3_div2); - const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + - im2 * mask->nb[2] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); + const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + + im2 * mask->nb[2] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); uint8_t * ms_dst = (uint8_t *) factx->vtcm_mask_buf + g * m_line_bytes; - dma_queue_push(dma, dma_make_ptr(ms_dst, ms_src), G * m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q); + dma_queue_push(dma_q, dma_make_data(ms_dst, ms_src), G * m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q); } } -static void fa_pop_mask_dma_gqa(dma_queue * dma, uint32_t G) { +static void fa_pop_mask_dma_gqa(dma_queue * dma_q, uint32_t G) { for (uint32_t g = 0; g < G; ++g) { - dma_queue_pop(dma); + dma_queue_pop(dma_q); } } -static inline void fa_prefetch_block(dma_queue * dma, const struct htp_tensor * k, const struct htp_tensor * v, const struct htp_tensor * mask, +static inline void fa_prefetch_block(dma_queue * dma_q, const struct htp_tensor * k, const struct htp_tensor * v, const struct htp_tensor * mask, uint32_t b, size_t Bc, size_t size_k_row_padded, size_t size_k_row, size_t size_v_row_padded, size_t size_v_row, uint32_t ik2, uint32_t ik3, uint32_t iv2, uint32_t iv3, uint32_t q_start, uint32_t im3, uint32_t kv_head, uint32_t G, size_t m_line_bytes, size_t n_rows_q, size_t nek1, size_t prefetch_buf, struct hmx_fa_context * factx) { const uint32_t prefetch_start = b * Bc; const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start); - const uint8_t * k_prefetch_src = (const uint8_t *) k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx->vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows); - const uint8_t * v_prefetch_src = (const uint8_t *) v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx->vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows); + const dma_addr_t k_prefetch_src = k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx->vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows); + const dma_addr_t v_prefetch_src = v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx->vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows); if (mask) { if (__builtin_expect(factx->mask_broadcast, true)) { - const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + prefetch_start * sizeof(__fp16); - dma_cache_push(dma, &factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q); + const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + prefetch_start * sizeof(__fp16); + dma_cache_fa_push(dma_q, &factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q); } else { - fa_push_mask_dma_gqa(dma, mask, q_start, im3, prefetch_start, kv_head, G, m_line_bytes, prefetch_rows, n_rows_q, factx); + fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, prefetch_start, kv_head, G, m_line_bytes, prefetch_rows, n_rows_q, factx); } } } @@ -1832,8 +1836,11 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const uint32_t DK = neq0; const uint32_t DV = nev0; - // HMX requires head_dim to be multiple of 32 - if (DK % 32 != 0 || DV % 32 != 0) { + // HMX tiles head_dim in units of 64. head_dim need not be 64- (or 32-) aligned: + // we can operate on DK/DV rounded up to 64 with tail lanes [D, D_pad) zero-filled. + const uint32_t DK_pad = hex_round_up(DK, 64); + const uint32_t DV_pad = hex_round_up(DV, 64); + if (DK == 0 || DV == 0) { return HTP_STATUS_NO_SUPPORT; } @@ -1850,6 +1857,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { factx.n_threads = kparams->n_threads; factx.DK = DK; factx.DV = DV; + factx.DK_pad = DK_pad; + factx.DV_pad = DV_pad; factx.n_kv = nek1; factx.n_kv_heads = n_kv_heads; factx.n_heads = neq2; @@ -1869,13 +1878,14 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { factx.src3_div3 = kparams->src3_div3; } - if (kparams->logit_softcap == 0.0f) { + factx.has_softcap = (kparams->logit_softcap != 0.0f); + if (!factx.has_softcap) { factx.scale = (__fp16) (kparams->scale * EXP_LOG2E_F); // log2(e) } else { factx.scale = (__fp16) kparams->scale; } factx.max_bias = kparams->max_bias; - factx.logit_softcap = (__fp16) (kparams->logit_softcap * EXP_LOG2E_F); + factx.logit_softcap = factx.has_softcap ? (__fp16) (kparams->logit_softcap * EXP_LOG2E_F) : 0; factx.n_head_log2 = kparams->n_head_log2; factx.m0 = kparams->m0; @@ -1888,18 +1898,38 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const uint32_t n_threads = factx.n_threads; const uint32_t G = factx.G; + // Multi-device: split Q blocks across devices + const uint32_t n_q_blocks = (neq1 + Br - 1) / Br; + uint32_t q_start_min = 0; + uint32_t q_start_max = neq1; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_q_blocks, htp_tensor_mdev_data_aligned(dst) ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + const uint32_t block_start = range.start; + const uint32_t block_end = range.start + range.count; + + if (block_start >= block_end) { + return HTP_STATUS_OK; + } + + q_start_min = block_start * Br; + q_start_max = MIN(block_end * Br, neq1); + } + // ======== VTCM allocation (GQA-aware) ======== // K/V row sizes drive the DMA descriptors (not the VTCM layout) and are used - // throughout the KV loop below. + // throughout the KV loop below. The DMA copies only the real DK/DV columns; the + // staging rows are padded to hold DK_pad/DV_pad columns (tail zero-filled below) + // so the HMX interleave/tile logic can operate on 64-aligned head dims. const size_t size_k_row = htp_tensor_get_row_size(k->type, DK); const size_t size_v_row = htp_tensor_get_row_size(v->type, DV); - const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128); - const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128); + const size_t size_k_row_padded = hex_round_up(DK_pad * sizeof(__fp16), 128); + const size_t size_v_row_padded = hex_round_up(DV_pad * sizeof(__fp16), 128); // Build the VTCM layout once (shared with the host estimator) and place every - // scratch buffer at its computed offset. + // scratch buffer at its computed offset. Padded head dims size the HMX tiles. struct hmx_fa_vtcm_layout L; - hmx_fa_vtcm_layout_build(&L, G, DK, DV, Br, Bc, n_threads, pipeline, factx.is_q_fp32); + hmx_fa_vtcm_layout_build(&L, G, DK_pad, DV_pad, Br, Bc, n_threads, pipeline, factx.is_q_fp32, factx.sinks != NULL, factx.n_heads); if (L.total_bytes > ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; @@ -1941,22 +1971,41 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { factx.col_vec_bytes = L.col_vec_bytes; factx.d_tile_bytes = L.d_tile_bytes; factx.vtcm_slopes = VTCM_LAYOUT_PTR(__fp16, base, L.off_slopes); + factx.vtcm_sinks = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_sinks, factx.sinks != NULL); const size_t m_line_bytes = L.m_line_bytes; // used by the mask DMAs in the KV loop - dma_cache_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE); + dma_cache_fa_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE); + + // Head-dim padding: the K/V DMA staging buffers and the flat-Q buffer are laid out + // with padded row strides (size_{k,v,q}_row_padded, covering D_pad columns) but the + // DMA only writes the real D columns per row. Zero the whole staging buffers once up + // front so tail lanes [D, D_pad) stay zero for all KV blocks. No-op when already aligned. + if (DK_pad != DK || DV_pad != DV) { + const size_t k_buf_bytes = (size_t) factx.Bc * size_k_row_padded; + const size_t v_buf_bytes = (size_t) factx.Bc * size_v_row_padded; + hvx_splat_u8_a((char *) factx.vtcm_k_fp16[0], 0, k_buf_bytes); + hvx_splat_u8_a((char *) factx.vtcm_k_fp16[1], 0, k_buf_bytes); + hvx_splat_u8_a((char *) factx.vtcm_v_fp16[0], 0, v_buf_bytes); + hvx_splat_u8_a((char *) factx.vtcm_v_fp16[1], 0, v_buf_bytes); + // Flat-Q DMA scratch + if (factx.vtcm_q_dma) { + const size_t q_dma_bytes = hex_align_up(factx.g_br * DK * (factx.is_q_fp32 ? sizeof(float) : sizeof(__fp16)), 128); + hvx_splat_u8_a((char *) factx.vtcm_q_dma, 0, q_dma_bytes); + } + } // ======== Initialize HMX output scales ======== hmx_init_column_scales(factx.vtcm_hmx_scales_id, Q6_V_vsplat_R(0x3c00)); // 1.0 hmx_init_column_scales(factx.vtcm_hmx_scales_qk, hvx_vec_splat_f16(factx.scale)); - // ======== Skip compute if profiling ======== - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; - } - // ======== DMA setup ======== - dma_queue * const dma = ctx->dma[0]; + dma_queue * const dma_q = ctx->dma[0]; + + if (factx.sinks) { + dma_queue_push(dma_q, dma_make_data(factx.vtcm_sinks, factx.sinks->data), L.sinks_bytes, 0, factx.sinks->size, 1); + dma_queue_pop(dma_q); + } const size_t n_row_tiles_g_br = g_br / HMX_FP16_TILE_N_ROWS; const size_t n_tiles_per_bc = Bc / HMX_FP16_TILE_N_COLS; @@ -1977,7 +2026,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // ======== Main loop ======== for (uint32_t ib3 = 0; ib3 < neq3; ++ib3) { const uint32_t im3 = mask ? fastmodulo(ib3, mask->ne[3], &factx.src3_div3) : 0; - for (uint32_t q_start = 0; q_start < neq1; q_start += Br) { + for (uint32_t q_start = q_start_min; q_start < q_start_max; q_start += Br) { const uint32_t n_rows_q = hex_smin(Br, neq1 - q_start); const size_t n_rows_g = n_rows_q * G; const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS); @@ -1991,32 +2040,33 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // 1. Push Q and KV DMAs for the very first iteration. // Subsequent iterations are enqueued early at the end of the previous iteration. - if (ib3 == 0 && q_start == 0 && kv_head == 0) { - const uint8_t * q_ptr = (const uint8_t *) q->data; + if (ib3 == 0 && q_start == q_start_min && kv_head == 0) { + const dma_addr_t q_ptr = q->data + q_start * q->nb[1] + + (kv_head * factx.G) * q->nb[2] + ib3 * q->nb[3]; const size_t q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed; const size_t n_rows = q_transposed ? factx.G : n_rows_q; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_q_dma, q_ptr), q_row_bytes, hex_smax(q_src_stride, q_row_bytes), q_row_bytes, n_rows); + dma_queue_push(dma_q, dma_make_data(factx.vtcm_q_dma, q_ptr), q_row_bytes, hex_smax(q_src_stride, q_row_bytes), q_row_bytes, n_rows); if (factx.n_kv_blocks > 0) { - const uint8_t * k_src = (const uint8_t *) k->data + ik2 * k->nb[2] + ik3 * k->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0); + const dma_addr_t k_src = k->data + ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0); - const uint8_t * v_src = (const uint8_t *) v->data + iv2 * v->nb[2] + iv3 * v->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0); + const dma_addr_t v_src = v->data + iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0); if (factx.pipeline && mask) { if (__builtin_expect(factx.mask_broadcast, true)) { - const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + 0; - dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), n_rows_q); + const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + 0; + dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), n_rows_q); } else { - fa_push_mask_dma_gqa(dma, mask, q_start, im3, 0, kv_head, G, m_line_bytes, kv_rows0, n_rows_q, &factx); + fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, 0, kv_head, G, m_line_bytes, kv_rows0, n_rows_q, &factx); } } } } // 2. Pop Q DMA (blocks until Q is loaded) - dma_queue_pop(dma); + dma_queue_pop(dma_q); // ---- Load Q block & Initialize per-block state ---- fa_phase_q_load(&factx, q, q_start, kv_head, ib3, n_rows_g); @@ -2043,12 +2093,12 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // Prefetch block 1 early if there are multiple blocks if (factx.n_kv_blocks > 1) { - fa_prefetch_block(dma, k, v, mask, 1, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row, + fa_prefetch_block(dma_q, k, v, mask, 1, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row, ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, 1, &factx); } // Prep and start QK-dot(0) - void * curr_k0 = dma_queue_pop(dma).dst; + void * curr_k0 = (void *) dma_queue_pop(dma_q).dst; fa_phase_k_interleave(&factx, kv_rows0, k_src_stride, curr_k0, 0, 0); qk_job[0].q_tiles = factx.vtcm_q_tiles; @@ -2056,7 +2106,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { qk_job[0].s_tiles = factx.vtcm_s_tiles[0]; qk_job[0].n_row_tiles = n_row_tiles; qk_job[0].n_col_tiles = hmx_ceil_div(kv_rows0, HMX_FP16_TILE_N_COLS); - qk_job[0].n_dot_tiles = DK / 32; + qk_job[0].n_dot_tiles = DK_pad / 32; qk_job[0].n_tiles_per_bc = n_tiles_per_bc; qk_job[0].hmx_scales = factx.vtcm_hmx_scales_qk; hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job[0])); @@ -2067,16 +2117,16 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS); // ---- 1. Pop and run V-prep for current block ---- - void * curr_v = dma_queue_pop(dma).dst; + void * curr_v = (void *) dma_queue_pop(dma_q).dst; fa_phase_v_interleave(&factx, kv_rows, v_src_stride, curr_v, factx.vtcm_v_tiles[buf_idx], n_tiles_per_bc, kv_start); // ---- 2. Pop and run mask-prep for current block ---- __fp16 * current_mask_vtcm = NULL; if (mask) { if (__builtin_expect(factx.mask_broadcast, true)) { - current_mask_vtcm = (__fp16 *) dma_queue_pop(dma).dst; + current_mask_vtcm = (__fp16 *) dma_queue_pop(dma_q).dst; } else { - fa_pop_mask_dma_gqa(dma, G); + fa_pop_mask_dma_gqa(dma_q, G); current_mask_vtcm = factx.vtcm_mask_buf; } } @@ -2100,7 +2150,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { hmx_ceil_div(hex_smin(Bc, nek1 - (kv_blk - 1) * Bc), HMX_FP16_TILE_N_COLS); ou_job[prev_buf].n_row_tiles_g_br = n_row_tiles_g_br; ou_job[prev_buf].n_tiles_per_bc = n_tiles_per_bc; - ou_job[prev_buf].DV = DV; + ou_job[prev_buf].DV = DV_pad; hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[prev_buf])); } @@ -2110,7 +2160,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const uint32_t next_rows = hex_smin(Bc, nek1 - next_start); const size_t next_buf = 1 - buf_idx; - void * next_k = dma_queue_pop(dma).dst; + void * next_k = (void *) dma_queue_pop(dma_q).dst; fa_phase_k_interleave(&factx, next_rows, k_src_stride, next_k, next_start, next_buf); qk_job[next_buf].q_tiles = factx.vtcm_q_tiles; @@ -2118,7 +2168,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { qk_job[next_buf].s_tiles = factx.vtcm_s_tiles[next_buf]; qk_job[next_buf].n_row_tiles = n_row_tiles; qk_job[next_buf].n_col_tiles = hmx_ceil_div(next_rows, HMX_FP16_TILE_N_COLS); - qk_job[next_buf].n_dot_tiles = DK / 32; + qk_job[next_buf].n_dot_tiles = DK_pad / 32; qk_job[next_buf].n_tiles_per_bc = n_tiles_per_bc; qk_job[next_buf].hmx_scales = factx.vtcm_hmx_scales_qk; hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job[next_buf])); @@ -2161,7 +2211,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // Prefetch block kv_blk + 2 if (kv_blk + 2 < factx.n_kv_blocks) { - fa_prefetch_block(dma, k, v, mask, kv_blk + 2, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row, + fa_prefetch_block(dma_q, k, v, mask, kv_blk + 2, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row, ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, buf_idx, &factx); } @@ -2182,7 +2232,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { ou_job[0].n_col_tiles = last_cols; ou_job[0].n_row_tiles_g_br = n_row_tiles_g_br; ou_job[0].n_tiles_per_bc = n_tiles_per_bc; - ou_job[0].DV = DV; + ou_job[0].DV = DV_pad; hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[0])); // Overlapped: run HVX build diag inv L while HMX is busy executing the update @@ -2203,10 +2253,10 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { if (mask) { if (__builtin_expect(factx.mask_broadcast, true)) { - const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); - dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q); + const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); + dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q); } else { - fa_push_mask_dma_gqa(dma, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx); + fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx); } } @@ -2214,14 +2264,14 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const uint32_t prefetch_start = (kv_blk + 1) * Bc; const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start); const size_t prefetch_buf = 1 - buf_idx; - const uint8_t * k_prefetch_src = (const uint8_t *) k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows); - const uint8_t * v_prefetch_src = (const uint8_t *) v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows); + const dma_addr_t k_prefetch_src = k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows); + const dma_addr_t v_prefetch_src = v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows); } // Wait for current K DMA and interleave - void * curr_k = dma_queue_pop(dma).dst; + void * curr_k = (void *) dma_queue_pop(dma_q).dst; fa_phase_k_interleave(&factx, kv_rows, k_src_stride, curr_k, kv_start, 0); { @@ -2230,7 +2280,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { qk_job.s_tiles = factx.vtcm_s_tiles[0]; qk_job.n_row_tiles = n_row_tiles; qk_job.n_col_tiles = n_col_tiles; - qk_job.n_dot_tiles = (size_t) (DK / 32); + qk_job.n_dot_tiles = (size_t) (DK_pad / 32); qk_job.n_tiles_per_bc = n_tiles_per_bc; qk_job.hmx_scales = factx.vtcm_hmx_scales_qk; @@ -2239,16 +2289,16 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { } // Wait for current V DMA and interleave - void * curr_v = dma_queue_pop(dma).dst; + void * curr_v = (void *) dma_queue_pop(dma_q).dst; fa_phase_v_interleave(&factx, kv_rows, v_src_stride, curr_v, factx.vtcm_v_tiles[0], n_tiles_per_bc, kv_start); // ---- Phase 3: softmax + build_D ---- __fp16 * current_mask_vtcm = NULL; if (mask) { if (__builtin_expect(factx.mask_broadcast, true)) { - current_mask_vtcm = (__fp16 *) dma_queue_pop(dma).dst; + current_mask_vtcm = (__fp16 *) dma_queue_pop(dma_q).dst; } else { - fa_pop_mask_dma_gqa(dma, G); + fa_pop_mask_dma_gqa(dma_q, G); current_mask_vtcm = factx.vtcm_mask_buf; } } @@ -2286,7 +2336,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { ou_job.n_col_tiles = n_col_tiles; ou_job.n_row_tiles_g_br = n_row_tiles_g_br; ou_job.n_tiles_per_bc = n_tiles_per_bc; - ou_job.DV = DV; + ou_job.DV = DV_pad; hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job)); if (kv_blk + 1 == factx.n_kv_blocks) { @@ -2311,8 +2361,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { if (next_kv_head >= n_kv_heads) { next_kv_head = 0; next_q_start = q_start + Br; - if (next_q_start >= neq1) { - next_q_start = 0; + if (next_q_start >= q_start_max) { + next_q_start = q_start_min; next_ib3 = ib3 + 1; } } @@ -2320,10 +2370,10 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { if (has_next) { const uint32_t next_n_rows_q = hex_smin(Br, neq1 - next_q_start); - const uint8_t * next_q_ptr = (const uint8_t *) q->data + next_q_start * q->nb[1] + (next_kv_head * factx.G) * q->nb[2] + next_ib3 * q->nb[3]; + const dma_addr_t next_q_ptr = q->data + next_q_start * q->nb[1] + (next_kv_head * factx.G) * q->nb[2] + next_ib3 * q->nb[3]; const size_t next_q_row_bytes = q_transposed ? next_n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed; const size_t next_n_rows = q_transposed ? factx.G : next_n_rows_q; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_q_dma, next_q_ptr), next_q_row_bytes, hex_smax(q_src_stride, next_q_row_bytes), next_q_row_bytes, next_n_rows); + dma_queue_push(dma_q, dma_make_data(factx.vtcm_q_dma, next_q_ptr), next_q_row_bytes, hex_smax(q_src_stride, next_q_row_bytes), next_q_row_bytes, next_n_rows); if (factx.n_kv_blocks > 0) { const uint32_t next_ik2 = next_kv_head; @@ -2335,11 +2385,11 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { next_iv3 = fastdiv(next_ib3, &kparams->broadcast_rv3); } - const uint8_t * next_k_src = (const uint8_t *) k->data + next_ik2 * k->nb[2] + next_ik3 * k->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[0], next_k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0); + const dma_addr_t next_k_src = k->data + next_ik2 * k->nb[2] + next_ik3 * k->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], next_k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0); - const uint8_t * next_v_src = (const uint8_t *) v->data + next_iv2 * v->nb[2] + next_iv3 * v->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[0], next_v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0); + const dma_addr_t next_v_src = v->data + next_iv2 * v->nb[2] + next_iv3 * v->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], next_v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0); if (factx.pipeline && mask) { uint32_t next_im3 = im3; @@ -2347,10 +2397,10 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { next_im3 = fastmodulo(next_ib3, mask->ne[3], &factx.src3_div3); } if (__builtin_expect(factx.mask_broadcast, true)) { - const uint8_t * ms_src = (const uint8_t *) mask->data + next_q_start * mask->nb[1] + next_im3 * mask->nb[3] + 0; - dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), next_n_rows_q); + const dma_addr_t ms_src = mask->data + next_q_start * mask->nb[1] + next_im3 * mask->nb[3] + 0; + dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), next_n_rows_q); } else { - fa_push_mask_dma_gqa(dma, mask, next_q_start, next_im3, 0, next_kv_head, G, m_line_bytes, kv_rows0, next_n_rows_q, &factx); + fa_push_mask_dma_gqa(dma_q, mask, next_q_start, next_im3, 0, next_kv_head, G, m_line_bytes, kv_rows0, next_n_rows_q, &factx); } } } @@ -2364,7 +2414,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { on_job.hmx_scales = factx.vtcm_hmx_scales_id; on_job.n_row_tiles = n_row_tiles; on_job.n_row_tiles_g_br = n_row_tiles_g_br; - on_job.DV = DV; + on_job.DV = DV_pad; hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_norm_worker, &on_job)); hmx_queue_pop(ctx->hmx_queue); } @@ -2392,12 +2442,20 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } + if (htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params; if (kparams->kernel_type == HTP_FA_KERNEL_UNSUPPORTED) { return HTP_STATUS_NO_SUPPORT; } + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + if (kparams->kernel_type == HTP_FA_KERNEL_HMX) { return hmx_flash_attn_ext(octx); } @@ -2407,8 +2465,6 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { factx.k = k; factx.v = v; - factx.t_start = HAP_perf_get_qtimer_count(); - factx.src0_div21 = kparams->u.hvx.src0_div21; factx.src0_div1 = kparams->u.hvx.src0_div1; @@ -2427,16 +2483,12 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { factx.size_k_row_padded = kparams->u.hvx.size_k_row_padded; factx.size_v_row_padded = kparams->u.hvx.size_v_row_padded; - size_t size_q_block = factx.size_q_row_padded * 1; // single row for now - factx.size_k_block = factx.size_k_row_padded * FLASH_ATTN_BLOCK_SIZE; - factx.size_v_block = factx.size_v_row_padded * FLASH_ATTN_BLOCK_SIZE; - factx.size_m_block = hex_round_up(FLASH_ATTN_BLOCK_SIZE * sizeof(__fp16), 128); - factx.n_blocks = kparams->n_kv_blocks; factx.scale = kparams->scale; factx.max_bias = kparams->max_bias; - factx.logit_softcap = (__fp16) kparams->logit_softcap; + factx.has_softcap = (kparams->logit_softcap != 0.0f); + factx.logit_softcap = factx.has_softcap ? (__fp16) kparams->logit_softcap : 0; factx.n_head_log2 = kparams->n_head_log2; factx.m0 = kparams->m0; @@ -2451,29 +2503,63 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { } // total rows in q - factx.qrows = kparams->qrows; - factx.qrows_per_thread = kparams->qrows_per_thread; + const uint32_t neq1 = q->ne[1]; + const uint32_t neq2 = q->ne[2]; + const uint32_t neq3 = q->ne[3]; + const uint32_t total_qrows = neq1 * neq2 * neq3; - size_t size_vkq_acc = hex_round_up(v->ne[0] * sizeof(float), 128); // VKQ32 + uint32_t qrow_start = 0; + uint32_t qrows = total_qrows; - factx.size_q_block = size_q_block; - factx.size_vkq_acc = size_vkq_acc; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_qrows, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + qrow_start = range.start; + qrows = range.count; + } - uint8_t * vtcm_cur = octx->ctx->vtcm_base; + if (qrows == 0) { + return HTP_STATUS_OK; + } - factx.spad_q = vtcm_seq_alloc(&vtcm_cur, size_q_block * octx->n_threads); - factx.spad_k = vtcm_seq_alloc(&vtcm_cur, factx.size_k_block * 2 * octx->n_threads); - factx.spad_v = vtcm_seq_alloc(&vtcm_cur, factx.size_v_block * 2 * octx->n_threads); - factx.spad_m = vtcm_seq_alloc(&vtcm_cur, (mask ? factx.size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * octx->n_threads); - factx.spad_a = vtcm_seq_alloc(&vtcm_cur, size_vkq_acc * octx->n_threads); + const uint32_t n_threads = octx->n_threads; - if ((size_t) (vtcm_cur - octx->ctx->vtcm_base) > octx->ctx->vtcm_size) { + factx.qrows = qrows; + factx.qrow_start = qrow_start; + factx.qrows_per_thread = fastdiv(qrows + n_threads - 1, &octx->n_threads_div); + + const bool has_mask = (mask != NULL); + const bool has_sinks = (octx->src[4] != NULL); + struct hvx_fa_vtcm_layout L; + hvx_fa_vtcm_layout_build(&L, k->ne[0], v->ne[0], factx.is_q_fp32, has_mask, has_sinks, n_head, n_threads); + + if (L.total_bytes > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; } - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, octx->n_threads); + factx.size_q_block = L.size_q_block; + factx.size_k_block = L.size_k_block; + factx.size_v_block = L.size_v_block; + factx.size_m_block = L.size_m_block; + factx.size_vkq_acc = L.size_vkq_acc; + + uint8_t * const base = octx->ctx->vtcm_base; + + factx.spad_q = VTCM_LAYOUT_PTR(uint8_t, base, L.off_q); + factx.spad_k = VTCM_LAYOUT_PTR(uint8_t, base, L.off_k); + factx.spad_v = VTCM_LAYOUT_PTR(uint8_t, base, L.off_v); + factx.spad_m = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_m, has_mask); + factx.spad_a = VTCM_LAYOUT_PTR(uint8_t, base, L.off_a); + factx.spad_sinks = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_sinks, has_sinks); + + if (has_sinks) { + const struct htp_tensor * sinks = octx->src[4]; + dma_queue * dma_q = octx->ctx->dma[0]; + dma_queue_push(dma_q, dma_make_data(factx.spad_sinks, sinks->data), L.size_sinks, 0, sinks->size, 1); + dma_queue_pop(dma_q); } + work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, n_threads); + return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h index c4d19063169b..22bb8c53d340 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h @@ -51,6 +51,7 @@ struct htp_fa_kernel_params { uint32_t qrows; uint32_t qrows_per_thread; + uint32_t qrow_start; float m0; float m1; uint32_t n_head_log2; @@ -120,6 +121,7 @@ struct hmx_fa_vtcm_layout { size_t off_hmx_scales_qk; size_t off_mask_buf; size_t off_slopes; + size_t off_sinks; // Region byte sizes reused by the device at runtime (not just for allocation). size_t q_tile_bytes; @@ -129,6 +131,7 @@ struct hmx_fa_vtcm_layout { size_t m_line_bytes; // one mask row size_t m_buf_slot_bytes; // one dma_cache slot = align_up(Br * m_line_bytes, 4096) size_t col_vec_bytes; + size_t sinks_bytes; // Derived strides. size_t row_buf_stride; // HVX vectors (128B) per row buffer @@ -141,8 +144,10 @@ struct hmx_fa_vtcm_layout { // Build the VTCM layout. static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, - size_t gqa_factor, size_t DK, size_t DV, - size_t Br, size_t Bc, size_t n_threads, bool pipeline, bool is_q_fp32) { + size_t gqa_factor, size_t DK, size_t DV, + size_t Br, size_t Bc, size_t n_threads, + bool pipeline, bool is_q_fp32, + bool has_sinks, size_t n_heads) { const size_t g_br = hex_align_up(gqa_factor * Br, HMX_FP16_TILE_N_ROWS); const size_t q_tile_size = hex_align_up(g_br * DK * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); const size_t o_tile_size = hex_align_up(g_br * DV * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); @@ -165,6 +170,7 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, const size_t m_buf_slot = hex_align_up(Br * m_line_size, 256); const size_t m_buf_size = m_buf_slot * HMX_FA_DMA_CACHE_SIZE; const size_t slopes_size = hex_align_up(g_br * sizeof(__fp16), 128); + const size_t sinks_size = hex_round_up(n_heads * sizeof(float), 128); size_t off = 0; @@ -214,6 +220,7 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, VTCM_LAYOUT_ALLOC(off, off_hmx_scales_qk, 256); VTCM_LAYOUT_ALLOC(off, off_mask_buf, m_buf_size); VTCM_LAYOUT_ALLOC(off, off_slopes, slopes_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_sinks, sinks_size, has_sinks); L->q_tile_bytes = q_tile_size; L->o_tile_bytes = o_tile_size; @@ -227,52 +234,94 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, L->m_buf_slot_bytes = m_buf_slot; L->row_buf_stride = row_vec_size / 128; L->mask_buf_row_stride = m_line_size / sizeof(__fp16); + L->sinks_bytes = has_sinks ? sinks_size : 0; L->pipeline = pipeline; L->total_bytes = off; } // Exact VTCM usage for a given (gqa_factor, DK, DV, Br, Bc) configuration. -static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_threads, bool pipeline, bool is_q_fp32) { +static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_threads, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) { struct hmx_fa_vtcm_layout L; - hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_threads, pipeline, is_q_fp32); + hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_threads, pipeline, is_q_fp32, has_sinks, n_heads); return L.total_bytes; } #define FA_HVX_BLOCK_SIZE 64 +#define FA_HVX_G_MAX 8 + +struct hvx_fa_vtcm_layout { + size_t off_q; + size_t off_k; + size_t off_v; + size_t off_m; + size_t off_a; + size_t off_sinks; + + size_t size_q_block; + size_t size_k_block; + size_t size_v_block; + size_t size_m_block; + size_t size_vkq_acc; + size_t size_sinks; -static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, size_t n_threads) { + size_t total_bytes; +}; + +static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L, + size_t DK, size_t DV, + bool is_q_fp32, bool has_mask, + bool has_sinks, size_t n_heads, + size_t n_threads) { const size_t size_q_row_padded = hex_round_up(DK * (is_q_fp32 ? 4 : 2), 128); const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128); const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128); - const size_t size_q_block = size_q_row_padded * 1; + const size_t size_q_block = size_q_row_padded * FA_HVX_G_MAX; const size_t size_k_block = size_k_row_padded * FA_HVX_BLOCK_SIZE; const size_t size_v_block = size_v_row_padded * FA_HVX_BLOCK_SIZE; const size_t size_m_block = hex_round_up(FA_HVX_BLOCK_SIZE * sizeof(__fp16), 128); - const size_t size_vkq_acc = hex_round_up(DV * sizeof(float), 128); + const size_t size_vkq_acc = hex_round_up(DV * sizeof(float), 128) * FA_HVX_G_MAX; + const size_t size_sinks = hex_round_up(n_heads * sizeof(float), 128); - const size_t size_per_thread = size_q_block * 1 - + size_k_block * 2 - + size_v_block * 2 - + (has_mask ? size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) - + size_vkq_acc; + size_t off = 0; - return size_per_thread * n_threads; + VTCM_LAYOUT_ALLOC(off, off_q, size_q_block * n_threads); + VTCM_LAYOUT_ALLOC(off, off_k, size_k_block * 2 * n_threads); + VTCM_LAYOUT_ALLOC(off, off_v, size_v_block * 2 * n_threads); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_m, size_m_block * HVX_FA_DMA_CACHE_SIZE * n_threads, has_mask); + VTCM_LAYOUT_ALLOC(off, off_a, size_vkq_acc * n_threads); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_sinks, size_sinks, has_sinks); + + L->size_q_block = size_q_block; + L->size_k_block = size_k_block; + L->size_v_block = size_v_block; + L->size_m_block = size_m_block; + L->size_vkq_acc = size_vkq_acc; + L->size_sinks = has_sinks ? size_sinks : 0; + L->total_bytes = off; +} + +static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, size_t n_threads) { + struct hvx_fa_vtcm_layout L; + hvx_fa_vtcm_layout_build(&L, DK, DV, is_q_fp32, has_mask, has_sinks, n_heads, n_threads); + return L.total_bytes; } #define FA_MIN_KV_BLOCKS 3 // Cost-based (Br, Bc) search for flash attention with pipeline constraint. static inline int hmx_fa_find_chunk_size(size_t * Br_out, - size_t * Bc_out, - size_t gqa_factor, - size_t DK, - size_t DV, - size_t qo_len, - size_t kv_len, - size_t vtcm_budget, - size_t n_threads, - bool is_q_fp32) { + size_t * Bc_out, + size_t gqa_factor, + size_t DK, + size_t DV, + size_t qo_len, + size_t kv_len, + size_t vtcm_budget, + size_t n_threads, + bool is_q_fp32, + bool has_sinks, + size_t n_heads) { const size_t T = HMX_FP16_TILE_N_ROWS; // 32 const size_t br_unit = hmx_ceil_div(T, gqa_factor); const size_t bc_unit = HMX_FP16_TILE_N_COLS * 2; // 64 @@ -296,7 +345,7 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out, for (size_t Br = Br_max; Br >= br_unit; Br -= br_unit) { // Try all Bc candidates from Bc_limit down to bc_unit for (size_t Bc = Bc_limit; Bc >= bc_unit; Bc -= bc_unit) { - size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_threads, can_pipeline, is_q_fp32); + size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_threads, can_pipeline, is_q_fp32, has_sinks, n_heads); if (vtcm_needed <= vtcm_budget) { // This Bc fits for this Br! const size_t q_blocks = (qo_len + Br - 1) / Br; diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c index 96655215298a..1dd828db7b84 100644 --- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c @@ -1,39 +1,45 @@ -#include <math.h> #include <stdint.h> +#include <stdbool.h> #include <string.h> +#include <math.h> +#include <HAP_farf.h> -#include "hvx-utils.h" -#include "hex-fastdiv.h" - -#define GGML_COMMON_DECL_C +#include "hvx-base.h" +#include "hvx-copy.h" +#include "hvx-reduce.h" +#include "hvx-exp.h" +#include "dma-queue.h" #include "ggml-common.h" #include "htp-ctx.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "hmx-utils.h" +#include "hmx-fa-kernels.h" +#include "hmx-queue.h" +#include "gated-delta-net-ops.h" #ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) #endif -#define HTP_GDN_MAX_SV 128 - - struct htp_gdn_context { struct htp_ops_context * octx; - uint32_t rows_per_thread; - size_t state_bytes; + const struct htp_gdn_kernel_params * kparams; + struct htp_gdn_vtcm_layout layout; uint8_t * vtcm_base; - size_t vtcm_per_thread; + uint32_t row_start; + uint32_t nrows; }; -static inline HVX_Vector gdn_mul_dot_f32(float * restrict dst, const float * restrict mul, const float * restrict dot, uint32_t n) { +static inline HVX_Vector gdn_mul_dot_f32(float * restrict dst, const HVX_Vector * restrict mul, const HVX_Vector * restrict dot, uint32_t n) { HVX_Vector acc = Q6_V_vzero(); - - const uint32_t epv = 128 / sizeof(float); + const uint32_t epv = 128 / sizeof(float); const uint32_t nvec = n / epv; const uint32_t nloe = n % epv; for (uint32_t i = 0; i < nvec; ++i) { HVX_Vector vd = hvx_vmemu(dst + i * epv); - HVX_Vector vm = hvx_vmem(mul + i * epv); - HVX_Vector vdot = hvx_vmem(dot + i * epv); + HVX_Vector vm = mul[i]; + HVX_Vector vdot = dot[i]; HVX_Vector out = hvx_vec_mul_f32_f32(vd, vm); hvx_vmemu(dst + i * epv) = out; acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(out, vdot)); @@ -41,29 +47,27 @@ static inline HVX_Vector gdn_mul_dot_f32(float * restrict dst, const float * res if (nloe) { const uint32_t off = nvec * epv; - HVX_Vector vd = hvx_vmemu(dst + off); - HVX_Vector vm = hvx_vmem(mul + off); - HVX_Vector vdot = hvx_vmem(dot + off); - HVX_Vector out = hvx_vec_mul_f32_f32(vd, vm); - hvx_vec_store_u(dst + off, nloe * sizeof(float), out); + HVX_Vector vm = mul[nvec]; + HVX_Vector vdot = dot[nvec]; HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); - HVX_Vector prod = hvx_vec_mul_f32_f32(out, vdot); - acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, prod, Q6_V_vzero())); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out = hvx_vec_mul_f32_f32(hvx_vmemu(dst + off), vm); + hvx_vec_store_u(dst + off, nloe * sizeof(float), out); + acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out, vdot), zero)); } return hvx_vec_reduce_sum_f32(acc); } -static inline HVX_Vector gdn_mul_scalar_dot_f32(float * restrict dst, float mul, const float * restrict dot, uint32_t n) { +static inline HVX_Vector gdn_mul_scalar_dot_f32(float * restrict dst, HVX_Vector vmul, const HVX_Vector * restrict dot, uint32_t n) { HVX_Vector acc = Q6_V_vzero(); - const HVX_Vector vmul = hvx_vec_splat_f32(mul); - - const uint32_t epv = 128 / sizeof(float); + const uint32_t epv = 128 / sizeof(float); const uint32_t nvec = n / epv; const uint32_t nloe = n % epv; for (uint32_t i = 0; i < nvec; ++i) { HVX_Vector vd = hvx_vmemu(dst + i * epv); - HVX_Vector vdot = hvx_vmem(dot + i * epv); + HVX_Vector vdot = dot[i]; HVX_Vector out = hvx_vec_mul_f32_f32(vd, vmul); hvx_vmemu(dst + i * epv) = out; acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(out, vdot)); @@ -71,29 +75,28 @@ static inline HVX_Vector gdn_mul_scalar_dot_f32(float * restrict dst, float mul, if (nloe) { const uint32_t off = nvec * epv; - HVX_Vector vd = hvx_vmemu(dst + off); - HVX_Vector vdot = hvx_vmem(dot + off); - HVX_Vector out = hvx_vec_mul_f32_f32(vd, vmul); - hvx_vec_store_u(dst + off, nloe * sizeof(float), out); + HVX_Vector vdot = dot[nvec]; HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); - HVX_Vector prod = hvx_vec_mul_f32_f32(out, vdot); - acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, prod, Q6_V_vzero())); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out = hvx_vec_mul_f32_f32(hvx_vmemu(dst + off), vmul); + hvx_vec_store_u(dst + off, nloe * sizeof(float), out); + acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out, vdot), zero)); } return hvx_vec_reduce_sum_f32(acc); } -static inline HVX_Vector gdn_add_scaled_dot_f32(float * restrict dst, const float * restrict src, - HVX_Vector vscale, const float * restrict dot, uint32_t n) { +static inline HVX_Vector gdn_add_scaled_dot_f32(float * restrict dst, const HVX_Vector * restrict src, + HVX_Vector vscale, const HVX_Vector * restrict dot, uint32_t n) { HVX_Vector acc = Q6_V_vzero(); - - const uint32_t epv = 128 / sizeof(float); + const uint32_t epv = 128 / sizeof(float); const uint32_t nvec = n / epv; const uint32_t nloe = n % epv; for (uint32_t i = 0; i < nvec; ++i) { HVX_Vector vd = hvx_vmemu(dst + i * epv); - HVX_Vector vs = hvx_vmem(src + i * epv); - HVX_Vector vdot = hvx_vmem(dot + i * epv); + HVX_Vector vs = src[i]; + HVX_Vector vdot = dot[i]; HVX_Vector out = hvx_vec_add_f32_f32(vd, hvx_vec_mul_f32_f32(vs, vscale)); hvx_vmemu(dst + i * epv) = out; acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(out, vdot)); @@ -101,22 +104,22 @@ static inline HVX_Vector gdn_add_scaled_dot_f32(float * restrict dst, const floa if (nloe) { const uint32_t off = nvec * epv; - HVX_Vector vd = hvx_vmemu(dst + off); - HVX_Vector vs = hvx_vmem(src + off); - HVX_Vector vdot = hvx_vmem(dot + off); - HVX_Vector out = hvx_vec_add_f32_f32(vd, hvx_vec_mul_f32_f32(vs, vscale)); - hvx_vec_store_u(dst + off, nloe * sizeof(float), out); + HVX_Vector vs = src[nvec]; + HVX_Vector vdot = dot[nvec]; HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); - HVX_Vector prod = hvx_vec_mul_f32_f32(out, vdot); - acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, prod, Q6_V_vzero())); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out = hvx_vec_add_f32_f32(hvx_vmemu(dst + off), hvx_vec_mul_f32_f32(vs, vscale)); + hvx_vec_store_u(dst + off, nloe * sizeof(float), out); + acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out, vdot), zero)); } return hvx_vec_reduce_sum_f32(acc); } -static inline void gdn_mul_dot4_f32(float * restrict dst0, float * restrict dst1, - float * restrict dst2, float * restrict dst3, const float * restrict mul, - const float * restrict dot, uint32_t n, float * restrict sums) { +static inline HVX_Vector gdn_mul_dot4_f32(float * restrict dst0, float * restrict dst1, + float * restrict dst2, float * restrict dst3, + const HVX_Vector * restrict mul, const HVX_Vector * restrict dot, uint32_t n) { HVX_Vector acc0 = Q6_V_vzero(); HVX_Vector acc1 = Q6_V_vzero(); HVX_Vector acc2 = Q6_V_vzero(); @@ -126,8 +129,8 @@ static inline void gdn_mul_dot4_f32(float * restrict dst0, float * restrict dst1 const uint32_t nvec = n / epv; const uint32_t nloe = n % epv; for (uint32_t i = 0; i < nvec; ++i) { - HVX_Vector vm = hvx_vmem(mul + i * epv); - HVX_Vector vdot = hvx_vmem(dot + i * epv); + HVX_Vector vm = mul[i]; + HVX_Vector vdot = dot[i]; HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vm); HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vm); @@ -147,8 +150,8 @@ static inline void gdn_mul_dot4_f32(float * restrict dst0, float * restrict dst1 if (nloe) { const uint32_t off = nvec * epv; - HVX_Vector vm = hvx_vmem(mul + off); - HVX_Vector vdot = hvx_vmem(dot + off); + HVX_Vector vm = mul[nvec]; + HVX_Vector vdot = dot[nvec]; HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); HVX_Vector zero = Q6_V_vzero(); @@ -169,23 +172,22 @@ static inline void gdn_mul_dot4_f32(float * restrict dst0, float * restrict dst1 } HVX_Vector_x4 acc = { .v = { acc0, acc1, acc2, acc3 } }; - hvx_vec_store_u(sums, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(acc)); + return hvx_vec_reduce_sum_f32x4(acc); } -static inline void gdn_mul_scalar_dot4_f32(float * restrict dst0, float * restrict dst1, - float * restrict dst2, float * restrict dst3, float mul, - const float * restrict dot, uint32_t n, float * restrict sums) { +static inline HVX_Vector gdn_mul_scalar_dot4_f32(float * restrict dst0, float * restrict dst1, + float * restrict dst2, float * restrict dst3, + HVX_Vector vmul, const HVX_Vector * restrict dot, uint32_t n) { HVX_Vector acc0 = Q6_V_vzero(); HVX_Vector acc1 = Q6_V_vzero(); HVX_Vector acc2 = Q6_V_vzero(); HVX_Vector acc3 = Q6_V_vzero(); - const HVX_Vector vmul = hvx_vec_splat_f32(mul); const uint32_t epv = 128 / sizeof(float); const uint32_t nvec = n / epv; const uint32_t nloe = n % epv; for (uint32_t i = 0; i < nvec; ++i) { - HVX_Vector vdot = hvx_vmem(dot + i * epv); + HVX_Vector vdot = dot[i]; HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vmul); HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vmul); @@ -205,7 +207,7 @@ static inline void gdn_mul_scalar_dot4_f32(float * restrict dst0, float * restri if (nloe) { const uint32_t off = nvec * epv; - HVX_Vector vdot = hvx_vmem(dot + off); + HVX_Vector vdot = dot[nvec]; HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); HVX_Vector zero = Q6_V_vzero(); @@ -226,13 +228,13 @@ static inline void gdn_mul_scalar_dot4_f32(float * restrict dst0, float * restri } HVX_Vector_x4 acc = { .v = { acc0, acc1, acc2, acc3 } }; - hvx_vec_store_u(sums, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(acc)); + return hvx_vec_reduce_sum_f32x4(acc); } -static inline void gdn_add_scaled_dot4_f32(float * restrict dst0, float * restrict dst1, - float * restrict dst2, float * restrict dst3, const float * restrict src, - const float * restrict scale, const float * restrict dot, uint32_t n, - float * restrict sums) { +static inline HVX_Vector gdn_add_scaled_dot4_f32(float * restrict dst0, float * restrict dst1, + float * restrict dst2, float * restrict dst3, + const HVX_Vector * restrict src, const float * restrict scale, + const HVX_Vector * restrict dot, uint32_t n) { HVX_Vector acc0 = Q6_V_vzero(); HVX_Vector acc1 = Q6_V_vzero(); HVX_Vector acc2 = Q6_V_vzero(); @@ -246,8 +248,8 @@ static inline void gdn_add_scaled_dot4_f32(float * restrict dst0, float * restri const uint32_t nvec = n / epv; const uint32_t nloe = n % epv; for (uint32_t i = 0; i < nvec; ++i) { - HVX_Vector vs = hvx_vmem(src + i * epv); - HVX_Vector vdot = hvx_vmem(dot + i * epv); + HVX_Vector vs = src[i]; + HVX_Vector vdot = dot[i]; HVX_Vector out0 = hvx_vec_add_f32_f32(hvx_vmemu(dst0 + i * epv), hvx_vec_mul_f32_f32(vs, scale0)); HVX_Vector out1 = hvx_vec_add_f32_f32(hvx_vmemu(dst1 + i * epv), hvx_vec_mul_f32_f32(vs, scale1)); @@ -267,8 +269,8 @@ static inline void gdn_add_scaled_dot4_f32(float * restrict dst0, float * restri if (nloe) { const uint32_t off = nvec * epv; - HVX_Vector vs = hvx_vmem(src + off); - HVX_Vector vdot = hvx_vmem(dot + off); + HVX_Vector vs = src[nvec]; + HVX_Vector vdot = dot[nvec]; HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); HVX_Vector zero = Q6_V_vzero(); @@ -289,14 +291,13 @@ static inline void gdn_add_scaled_dot4_f32(float * restrict dst0, float * restri } HVX_Vector_x4 acc = { .v = { acc0, acc1, acc2, acc3 } }; - hvx_vec_store_u(sums, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(acc)); + return hvx_vec_reduce_sum_f32x4(acc); } -static inline void gdn_mul_dot8_f32(float * restrict dst0, float * restrict dst1, +static inline HVX_Vector gdn_mul_dot8_f32(float * restrict dst0, float * restrict dst1, float * restrict dst2, float * restrict dst3, float * restrict dst4, float * restrict dst5, float * restrict dst6, float * restrict dst7, - const float * restrict mul, const float * restrict dot, uint32_t n, - float * restrict sums) { + const HVX_Vector * restrict mul, const HVX_Vector * restrict dot, uint32_t n) { HVX_Vector acc0 = Q6_V_vzero(); HVX_Vector acc1 = Q6_V_vzero(); HVX_Vector acc2 = Q6_V_vzero(); @@ -310,8 +311,8 @@ static inline void gdn_mul_dot8_f32(float * restrict dst0, float * restrict dst1 const uint32_t nvec = n / epv; const uint32_t nloe = n % epv; for (uint32_t i = 0; i < nvec; ++i) { - HVX_Vector vm = hvx_vmem(mul + i * epv); - HVX_Vector vdot = hvx_vmem(dot + i * epv); + HVX_Vector vm = mul[i]; + HVX_Vector vdot = dot[i]; HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vm); HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vm); @@ -343,8 +344,8 @@ static inline void gdn_mul_dot8_f32(float * restrict dst0, float * restrict dst1 if (nloe) { const uint32_t off = nvec * epv; - HVX_Vector vm = hvx_vmem(mul + off); - HVX_Vector vdot = hvx_vmem(dot + off); + HVX_Vector vm = mul[nvec]; + HVX_Vector vdot = dot[nvec]; HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); HVX_Vector zero = Q6_V_vzero(); @@ -378,14 +379,16 @@ static inline void gdn_mul_dot8_f32(float * restrict dst0, float * restrict dst1 HVX_Vector_x4 accA = { .v = { acc0, acc1, acc2, acc3 } }; HVX_Vector_x4 accB = { .v = { acc4, acc5, acc6, acc7 } }; - hvx_vec_store_u(sums + 0, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accA)); - hvx_vec_store_u(sums + 4, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accB)); + HVX_Vector rA = hvx_vec_reduce_sum_f32x4(accA); + HVX_Vector rB = hvx_vec_reduce_sum_f32x4(accB); + HVX_VectorPred q16 = Q6_Q_vsetq2_R(16); + return Q6_V_vmux_QVV(q16, rA, Q6_V_vror_VR(rB, 128 - 16)); } -static inline void gdn_mul_scalar_dot8_f32(float * restrict dst0, float * restrict dst1, +static inline HVX_Vector gdn_mul_scalar_dot8_f32(float * restrict dst0, float * restrict dst1, float * restrict dst2, float * restrict dst3, float * restrict dst4, float * restrict dst5, float * restrict dst6, float * restrict dst7, - float mul, const float * restrict dot, uint32_t n, float * restrict sums) { + HVX_Vector vmul, const HVX_Vector * restrict dot, uint32_t n) { HVX_Vector acc0 = Q6_V_vzero(); HVX_Vector acc1 = Q6_V_vzero(); HVX_Vector acc2 = Q6_V_vzero(); @@ -394,13 +397,12 @@ static inline void gdn_mul_scalar_dot8_f32(float * restrict dst0, float * restri HVX_Vector acc5 = Q6_V_vzero(); HVX_Vector acc6 = Q6_V_vzero(); HVX_Vector acc7 = Q6_V_vzero(); - const HVX_Vector vmul = hvx_vec_splat_f32(mul); const uint32_t epv = 128 / sizeof(float); const uint32_t nvec = n / epv; const uint32_t nloe = n % epv; for (uint32_t i = 0; i < nvec; ++i) { - HVX_Vector vdot = hvx_vmem(dot + i * epv); + HVX_Vector vdot = dot[i]; HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vmul); HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vmul); @@ -432,7 +434,7 @@ static inline void gdn_mul_scalar_dot8_f32(float * restrict dst0, float * restri if (nloe) { const uint32_t off = nvec * epv; - HVX_Vector vdot = hvx_vmem(dot + off); + HVX_Vector vdot = dot[nvec]; HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); HVX_Vector zero = Q6_V_vzero(); @@ -466,15 +468,17 @@ static inline void gdn_mul_scalar_dot8_f32(float * restrict dst0, float * restri HVX_Vector_x4 accA = { .v = { acc0, acc1, acc2, acc3 } }; HVX_Vector_x4 accB = { .v = { acc4, acc5, acc6, acc7 } }; - hvx_vec_store_u(sums + 0, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accA)); - hvx_vec_store_u(sums + 4, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accB)); + HVX_Vector rA = hvx_vec_reduce_sum_f32x4(accA); + HVX_Vector rB = hvx_vec_reduce_sum_f32x4(accB); + HVX_VectorPred q16 = Q6_Q_vsetq2_R(16); + return Q6_V_vmux_QVV(q16, rA, Q6_V_vror_VR(rB, 128 - 16)); } -static inline void gdn_add_scaled_dot8_f32(float * restrict dst0, float * restrict dst1, +static inline HVX_Vector gdn_add_scaled_dot8_f32(float * restrict dst0, float * restrict dst1, float * restrict dst2, float * restrict dst3, float * restrict dst4, float * restrict dst5, float * restrict dst6, float * restrict dst7, - const float * restrict src, const float * restrict scale, - const float * restrict dot, uint32_t n, float * restrict sums) { + const HVX_Vector * restrict src, const float * restrict scale, + const HVX_Vector * restrict dot, uint32_t n) { HVX_Vector acc0 = Q6_V_vzero(); HVX_Vector acc1 = Q6_V_vzero(); HVX_Vector acc2 = Q6_V_vzero(); @@ -496,8 +500,8 @@ static inline void gdn_add_scaled_dot8_f32(float * restrict dst0, float * restri const uint32_t nvec = n / epv; const uint32_t nloe = n % epv; for (uint32_t i = 0; i < nvec; ++i) { - HVX_Vector vs = hvx_vmem(src + i * epv); - HVX_Vector vdot = hvx_vmem(dot + i * epv); + HVX_Vector vs = src[i]; + HVX_Vector vdot = dot[i]; HVX_Vector out0 = hvx_vec_add_f32_f32(hvx_vmemu(dst0 + i * epv), hvx_vec_mul_f32_f32(vs, scale0)); HVX_Vector out1 = hvx_vec_add_f32_f32(hvx_vmemu(dst1 + i * epv), hvx_vec_mul_f32_f32(vs, scale1)); @@ -529,8 +533,8 @@ static inline void gdn_add_scaled_dot8_f32(float * restrict dst0, float * restri if (nloe) { const uint32_t off = nvec * epv; - HVX_Vector vs = hvx_vmem(src + off); - HVX_Vector vdot = hvx_vmem(dot + off); + HVX_Vector vs = src[nvec]; + HVX_Vector vdot = dot[nvec]; HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); HVX_Vector zero = Q6_V_vzero(); @@ -564,13 +568,190 @@ static inline void gdn_add_scaled_dot8_f32(float * restrict dst0, float * restri HVX_Vector_x4 accA = { .v = { acc0, acc1, acc2, acc3 } }; HVX_Vector_x4 accB = { .v = { acc4, acc5, acc6, acc7 } }; - hvx_vec_store_u(sums + 0, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accA)); - hvx_vec_store_u(sums + 4, 4 * sizeof(float), hvx_vec_reduce_sum_f32x4(accB)); + HVX_Vector rA = hvx_vec_reduce_sum_f32x4(accA); + HVX_Vector rB = hvx_vec_reduce_sum_f32x4(accB); + HVX_VectorPred q16 = Q6_Q_vsetq2_R(16); + return Q6_V_vmux_QVV(q16, rA, Q6_V_vror_VR(rB, 128 - 16)); +} + +static inline void gdn_step_kda_f32( + float * restrict s_work, + float * restrict attn_out, + const float * restrict q_t, + const float * restrict k_t, + const float * restrict v_t, + const float * restrict g_t, + float beta_val, + float scale, + uint32_t S_v +) { + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = S_v / epv; + const uint32_t nloe = S_v % epv; + + HVX_Vector vq[4]; + HVX_Vector vk[4]; + HVX_Vector vg[4]; + + for (uint32_t i = 0; i < nvec; ++i) { + vq[i] = hvx_vmemu(q_t + i * epv); + vk[i] = hvx_vmemu(k_t + i * epv); + vg[i] = hvx_vec_exp_f32(hvx_vmemu(g_t + i * epv)); + } + if (nloe) { + vq[nvec] = hvx_vmemu(q_t + nvec * epv); + vk[nvec] = hvx_vmemu(k_t + nvec * epv); + vg[nvec] = hvx_vec_exp_f32(hvx_vmemu(g_t + nvec * epv)); + } + + const HVX_Vector vbeta = hvx_vec_splat_f32(beta_val); + const HVX_Vector vscale = hvx_vec_splat_f32(scale); + + float delta[8] __attribute__((aligned(128))); + + uint32_t j = 0; + for (; j + 8 <= S_v; j += 8) { + float * row0 = s_work + (uint64_t) (j + 0) * S_v; + float * row1 = s_work + (uint64_t) (j + 1) * S_v; + float * row2 = s_work + (uint64_t) (j + 2) * S_v; + float * row3 = s_work + (uint64_t) (j + 3) * S_v; + float * row4 = s_work + (uint64_t) (j + 4) * S_v; + float * row5 = s_work + (uint64_t) (j + 5) * S_v; + float * row6 = s_work + (uint64_t) (j + 6) * S_v; + float * row7 = s_work + (uint64_t) (j + 7) * S_v; + + HVX_Vector vsums = gdn_mul_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, + vg, vk, S_v); + + HVX_Vector vv_t = hvx_vmemu(v_t + j); + HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, vsums); + HVX_Vector vdelta = hvx_vec_mul_f32_f32(diff, vbeta); + hvx_vec_store_u(delta, 8 * sizeof(float), vdelta); + + HVX_Vector vattn = gdn_add_scaled_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, + vk, delta, vq, S_v); + + HVX_Vector res_attn = hvx_vec_mul_f32_f32(vattn, vscale); + hvx_vec_store_u(attn_out + j, 8 * sizeof(float), res_attn); + } + for (; j + 4 <= S_v; j += 4) { + float * row0 = s_work + (uint64_t) (j + 0) * S_v; + float * row1 = s_work + (uint64_t) (j + 1) * S_v; + float * row2 = s_work + (uint64_t) (j + 2) * S_v; + float * row3 = s_work + (uint64_t) (j + 3) * S_v; + + HVX_Vector vsums = gdn_mul_dot4_f32(row0, row1, row2, row3, vg, vk, S_v); + + HVX_Vector vv_t = hvx_vmemu(v_t + j); + HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, vsums); + HVX_Vector vdelta = hvx_vec_mul_f32_f32(diff, vbeta); + hvx_vec_store_u(delta, 4 * sizeof(float), vdelta); + + HVX_Vector vattn = gdn_add_scaled_dot4_f32(row0, row1, row2, row3, vk, delta, vq, S_v); + + HVX_Vector res_attn = hvx_vec_mul_f32_f32(vattn, vscale); + hvx_vec_store_u(attn_out + j, 4 * sizeof(float), res_attn); + } + for (; j < S_v; ++j) { + float * row = s_work + (uint64_t) j * S_v; + HVX_Vector vsum = gdn_mul_dot_f32(row, vg, vk, S_v); + HVX_Vector vv_t = hvx_vec_splat_f32(v_t[j]); + HVX_Vector vdj = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv_t, vsum), vbeta); + HVX_Vector vres = gdn_add_scaled_dot_f32(row, vk, vdj, vq, S_v); + attn_out[j] = hvx_vec_get_f32(hvx_vec_mul_f32_f32(vres, vscale)); + } +} + +static inline void gdn_step_scalar_f32( + float * restrict s_work, + float * restrict attn_out, + const float * restrict q_t, + const float * restrict k_t, + const float * restrict v_t, + const float * restrict g_t, + float beta_val, + float scale, + uint32_t S_v +) { + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = S_v / epv; + const uint32_t nloe = S_v % epv; + + HVX_Vector vq[4]; + HVX_Vector vk[4]; + + for (uint32_t i = 0; i < nvec; ++i) { + vq[i] = hvx_vmemu(q_t + i * epv); + vk[i] = hvx_vmemu(k_t + i * epv); + } + if (nloe) { + vq[nvec] = hvx_vmemu(q_t + nvec * epv); + vk[nvec] = hvx_vmemu(k_t + nvec * epv); + } + + const HVX_Vector vgate = hvx_vec_exp_f32(hvx_vec_splat_f32(g_t[0])); + const HVX_Vector vbeta = hvx_vec_splat_f32(beta_val); + const HVX_Vector vscale = hvx_vec_splat_f32(scale); + + float delta[8] __attribute__((aligned(128))); + + uint32_t j = 0; + for (; j + 8 <= S_v; j += 8) { + float * row0 = s_work + (uint64_t) (j + 0) * S_v; + float * row1 = s_work + (uint64_t) (j + 1) * S_v; + float * row2 = s_work + (uint64_t) (j + 2) * S_v; + float * row3 = s_work + (uint64_t) (j + 3) * S_v; + float * row4 = s_work + (uint64_t) (j + 4) * S_v; + float * row5 = s_work + (uint64_t) (j + 5) * S_v; + float * row6 = s_work + (uint64_t) (j + 6) * S_v; + float * row7 = s_work + (uint64_t) (j + 7) * S_v; + + HVX_Vector vsums = gdn_mul_scalar_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, + vgate, vk, S_v); + + HVX_Vector vv_t = hvx_vmemu(v_t + j); + HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, vsums); + HVX_Vector vdelta = hvx_vec_mul_f32_f32(diff, vbeta); + hvx_vec_store_u(delta, 8 * sizeof(float), vdelta); + + HVX_Vector vattn = gdn_add_scaled_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, + vk, delta, vq, S_v); + + HVX_Vector res_attn = hvx_vec_mul_f32_f32(vattn, vscale); + hvx_vec_store_u(attn_out + j, 8 * sizeof(float), res_attn); + } + for (; j + 4 <= S_v; j += 4) { + float * row0 = s_work + (uint64_t) (j + 0) * S_v; + float * row1 = s_work + (uint64_t) (j + 1) * S_v; + float * row2 = s_work + (uint64_t) (j + 2) * S_v; + float * row3 = s_work + (uint64_t) (j + 3) * S_v; + + HVX_Vector vsums = gdn_mul_scalar_dot4_f32(row0, row1, row2, row3, vgate, vk, S_v); + + HVX_Vector vv_t = hvx_vmemu(v_t + j); + HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, vsums); + HVX_Vector vdelta = hvx_vec_mul_f32_f32(diff, vbeta); + hvx_vec_store_u(delta, 4 * sizeof(float), vdelta); + + HVX_Vector vattn = gdn_add_scaled_dot4_f32(row0, row1, row2, row3, vk, delta, vq, S_v); + + HVX_Vector res_attn = hvx_vec_mul_f32_f32(vattn, vscale); + hvx_vec_store_u(attn_out + j, 4 * sizeof(float), res_attn); + } + for (; j < S_v; ++j) { + float * row = s_work + (uint64_t) j * S_v; + HVX_Vector vsum = gdn_mul_scalar_dot_f32(row, vgate, vk, S_v); + HVX_Vector vv_t = hvx_vec_splat_f32(v_t[j]); + HVX_Vector vdj = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv_t, vsum), vbeta); + HVX_Vector vres = gdn_add_scaled_dot_f32(row, vk, vdj, vq, S_v); + attn_out[j] = hvx_vec_get_f32(hvx_vec_mul_f32_f32(vres, vscale)); + } } static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, void * data) { struct htp_gdn_context * gctx = (struct htp_gdn_context *) data; struct htp_ops_context * octx = gctx->octx; + const struct htp_gdn_kernel_params * kparams = gctx->kparams; const struct htp_tensor * q = octx->src[0]; const struct htp_tensor * k = octx->src[1]; @@ -580,65 +761,55 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo const struct htp_tensor * state = octx->src[5]; const struct htp_tensor * dst = octx->dst; - const uint32_t S_v = v->ne[0]; - const uint32_t H = v->ne[1]; - const uint32_t n_tokens = v->ne[2]; - const uint32_t n_seqs = v->ne[3]; - const uint32_t K = octx->op_params[0]; + const uint32_t S_v = kparams->S_v; + const uint32_t H = kparams->H; + const uint32_t n_tokens = kparams->n_tokens; + const uint32_t n_seqs = kparams->n_seqs; + const uint32_t K = kparams->K; + const uint32_t row_end = gctx->row_start + gctx->nrows; - const uint32_t total_rows = H * n_seqs; - if (ith >= total_rows) { + if (ith >= gctx->nrows) { return; } - const uint32_t rq3 = n_seqs / q->ne[3]; - const uint32_t rk3 = n_seqs / k->ne[3]; - const float scale = 1.0f / sqrtf((float) S_v); - + const struct htp_tensor * dst_cache = octx->dsts[1]; + const float scale = kparams->scale; float * dst_base = (float *) (uintptr_t) dst->data; - float * state_out_base = dst_base + (uint64_t) S_v * H * n_tokens * n_seqs; - const float * state_in_base = (const float *) (uintptr_t) state->data; - - const bool kda = (g->ne[0] == S_v); - float local_gate[HTP_GDN_MAX_SV] __attribute__((aligned(128))); - float local_q[HTP_GDN_MAX_SV] __attribute__((aligned(128))); - float local_k[HTP_GDN_MAX_SV] __attribute__((aligned(128))); - float local_sums[32] __attribute__((aligned(128))); - - dma_queue * dma = octx->ctx->dma[ith]; - size_t state_aligned = (size_t) S_v * S_v * sizeof(float); - state_aligned = (state_aligned + 127) & ~(size_t)127; + float * state_out_base = dst_cache ? (float *) (uintptr_t) dst_cache->data : (dst_base + S_v * H * n_tokens * n_seqs); + + dma_queue * dma_q = octx->ctx->dma[ith]; + const struct htp_gdn_vtcm_layout * layout = &gctx->layout; float * s_work[2]; - s_work[0] = (float *) (gctx->vtcm_base + gctx->vtcm_per_thread * ith); - s_work[1] = s_work[0] + state_aligned / sizeof(float); + s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith); + s_work[1] = s_work[0] + layout->state_aligned / sizeof(float); - struct fastdiv_values fd_H = init_fastdiv_values(H); - struct fastdiv_values fd_q1 = init_fastdiv_values(q->ne[1]); - struct fastdiv_values fd_k1 = init_fastdiv_values(k->ne[1]); - struct fastdiv_values fd_rq3 = init_fastdiv_values(rq3); - struct fastdiv_values fd_rk3 = init_fastdiv_values(rk3); + const struct fastdiv_values * fd_H = &kparams->div_H; + const struct fastdiv_values * fd_q1 = &kparams->div_q1; + const struct fastdiv_values * fd_k1 = &kparams->div_k1; + const struct fastdiv_values * fd_rq3 = &kparams->div_rq3; + const struct fastdiv_values * fd_rk3 = &kparams->div_rk3; - const uint64_t state_seq_stride = state->nb[3] / sizeof(float); - const uint64_t state_size_per_snap = (uint64_t) S_v * S_v * H * n_seqs; + const uint32_t state_seq_stride = kparams->state_seq_stride; + const uint64_t state_size_per_snap = (uint64_t) kparams->state_size_per_snap; + const dma_addr_t state_out_dma_base = dst_cache ? dst_cache->data : (dst->data + S_v * H * n_tokens * n_seqs * sizeof(float)); - uint32_t ir_prefetch = ith; + uint32_t ir_prefetch = gctx->row_start + ith; int spad_idx = 0; // Prefetch preamble (up to 2 steps) - for (int k = 0; k < 2 && ir_prefetch < total_rows; k++) { - const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); - const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); - const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; - // final state lands in snapshot slot 0 (most-recent-first ordering) - float * ps_out = state_out_base + ((uint64_t) piv3 * H + piv1) * S_v * S_v; + for (int step = 0; step < 2 && ir_prefetch < row_end; step++) { + const uint32_t piv1 = fastmodulo(ir_prefetch, H, fd_H); + const uint32_t piv3 = fastdiv(ir_prefetch, fd_H); + dma_addr_t ps_in = state->data + ((uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v) * sizeof(float); + dma_addr_t ps_out = state_out_dma_base + ((uint64_t) piv3 * H + piv1) * S_v * S_v * sizeof(float); // Push dummy write-back - dma_queue_push(dma, dma_make_ptr(ps_out, s_work[spad_idx]), + dma_queue_push(dma_q, dma_make_data(ps_out, s_work[spad_idx]), S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), 0); // Push fetch - dma_queue_push(dma, dma_make_ptr(s_work[spad_idx], ps_in), + dma_queue_push(dma_q, dma_make_data(s_work[spad_idx], ps_in), S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), S_v); @@ -646,26 +817,27 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + int curr_spad_idx = 0; - for (uint32_t ir = ith; ir < total_rows; ir += nth) { - dma_queue_pop(dma); - dma_queue_pop(dma); + for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); float * s_work_curr = s_work[curr_spad_idx]; - const uint32_t iv1 = fastmodulo(ir, H, &fd_H); - const uint32_t iv3 = fastdiv(ir, &fd_H); - - const uint32_t iq1 = fastmodulo(iv1, q->ne[1], &fd_q1); - const uint32_t ik1 = fastmodulo(iv1, k->ne[1], &fd_k1); - const uint32_t iq3 = fastdiv(iv3, &fd_rq3); - const uint32_t ik3 = fastdiv(iv3, &fd_rk3); + const uint32_t iv1 = fastmodulo(ir, H, fd_H); + const uint32_t iv3 = fastdiv(ir, fd_H); - // final state lands in snapshot slot 0 (most-recent-first ordering) - float * s_out = state_out_base + ((uint64_t) iv3 * H + iv1) * S_v * S_v; + const uint32_t iq1 = fastmodulo(iv1, q->ne[1], fd_q1); + const uint32_t ik1 = fastmodulo(iv1, k->ne[1], fd_k1); + const uint32_t iq3 = fastdiv(iv3, fd_rq3); + const uint32_t ik3 = fastdiv(iv3, fd_rk3); + dma_addr_t s_out = state_out_dma_base + ((uint64_t) iv3 * H + iv1) * S_v * S_v * sizeof(float); float * attn_data = dst_base + ((uint64_t) iv3 * n_tokens * H + iv1) * S_v; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t t = 0; t < n_tokens; ++t) { const float * q_t = (const float *) ((const uint8_t *) (uintptr_t) q->data + (uint64_t) iq3 * q->nb[3] + (uint64_t) t * q->nb[2] + (uint64_t) iq1 * q->nb[1]); @@ -678,146 +850,36 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo const float beta_val = *(const float *) ((const uint8_t *) (uintptr_t) beta->data + (uint64_t) iv3 * beta->nb[3] + (uint64_t) t * beta->nb[2] + (uint64_t) iv1 * beta->nb[1]); - hvx_copy_f32_au((uint8_t *) local_q, (const uint8_t *) q_t, S_v); - hvx_copy_f32_au((uint8_t *) local_k, (const uint8_t *) k_t, S_v); - - if (kda) { - hvx_exp_f32((uint8_t *) local_gate, (const uint8_t *) g_t, S_v, false); - - uint32_t j = 0; - for (; j + 8 <= S_v; j += 8) { - float * row0 = s_work_curr + (uint64_t) (j + 0) * S_v; - float * row1 = s_work_curr + (uint64_t) (j + 1) * S_v; - float * row2 = s_work_curr + (uint64_t) (j + 2) * S_v; - float * row3 = s_work_curr + (uint64_t) (j + 3) * S_v; - float * row4 = s_work_curr + (uint64_t) (j + 4) * S_v; - float * row5 = s_work_curr + (uint64_t) (j + 5) * S_v; - float * row6 = s_work_curr + (uint64_t) (j + 6) * S_v; - float * row7 = s_work_curr + (uint64_t) (j + 7) * S_v; - gdn_mul_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, - local_gate, local_k, S_v, local_sums); - - float local_delta_b[32] __attribute__((aligned(128))); - HVX_Vector vv_t = hvx_vmemu(v_t + j); - HVX_Vector v_local_sums = hvx_vmem(local_sums); - HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, v_local_sums); - hvx_vmem(local_delta_b) = hvx_vec_mul_f32_f32(diff, hvx_vec_splat_f32(beta_val)); - - gdn_add_scaled_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, - local_k, local_delta_b, local_q, S_v, local_sums); - - HVX_Vector res_attn = hvx_vec_mul_f32_f32(hvx_vmem(local_sums), hvx_vec_splat_f32(scale)); - hvx_vec_store_u(attn_data + j, 8 * sizeof(float), res_attn); - } - for (; j + 4 <= S_v; j += 4) { - float * row0 = s_work_curr + (uint64_t) (j + 0) * S_v; - float * row1 = s_work_curr + (uint64_t) (j + 1) * S_v; - float * row2 = s_work_curr + (uint64_t) (j + 2) * S_v; - float * row3 = s_work_curr + (uint64_t) (j + 3) * S_v; - gdn_mul_dot4_f32(row0, row1, row2, row3, local_gate, local_k, S_v, local_sums); - - float local_delta_b[32] __attribute__((aligned(128))); - HVX_Vector vv_t = hvx_vmemu(v_t + j); - HVX_Vector v_local_sums = hvx_vmem(local_sums); - HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, v_local_sums); - hvx_vmem(local_delta_b) = hvx_vec_mul_f32_f32(diff, hvx_vec_splat_f32(beta_val)); - - gdn_add_scaled_dot4_f32(row0, row1, row2, row3, local_k, local_delta_b, local_q, S_v, local_sums); - - HVX_Vector res_attn = hvx_vec_mul_f32_f32(hvx_vmem(local_sums), hvx_vec_splat_f32(scale)); - hvx_vec_store_u(attn_data + j, 4 * sizeof(float), res_attn); - } - HVX_Vector vscale_splat = hvx_vec_splat_f32(scale); - for (; j < S_v; ++j) { - float * row = s_work_curr + (uint64_t) j * S_v; - HVX_Vector vsum = gdn_mul_dot_f32(row, local_gate, local_k, S_v); - HVX_Vector vv_t = hvx_vec_splat_f32(v_t[j]); - HVX_Vector vdj = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv_t, vsum), hvx_vec_splat_f32(beta_val)); - HVX_Vector vres = gdn_add_scaled_dot_f32(row, local_k, vdj, local_q, S_v); - attn_data[j] = hvx_vec_get_f32(hvx_vec_mul_f32_f32(vres, vscale_splat)); - } + if (kparams->kda) { + gdn_step_kda_f32(s_work_curr, attn_data, q_t, k_t, v_t, g_t, beta_val, scale, S_v); } else { - const float gate = expf(g_t[0]); - uint32_t j = 0; - for (; j + 8 <= S_v; j += 8) { - float * row0 = s_work_curr + (uint64_t) (j + 0) * S_v; - float * row1 = s_work_curr + (uint64_t) (j + 1) * S_v; - float * row2 = s_work_curr + (uint64_t) (j + 2) * S_v; - float * row3 = s_work_curr + (uint64_t) (j + 3) * S_v; - float * row4 = s_work_curr + (uint64_t) (j + 4) * S_v; - float * row5 = s_work_curr + (uint64_t) (j + 5) * S_v; - float * row6 = s_work_curr + (uint64_t) (j + 6) * S_v; - float * row7 = s_work_curr + (uint64_t) (j + 7) * S_v; - gdn_mul_scalar_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, - gate, local_k, S_v, local_sums); - - float local_delta_b[32] __attribute__((aligned(128))); - HVX_Vector vv_t = hvx_vmemu(v_t + j); - HVX_Vector v_local_sums = hvx_vmem(local_sums); - HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, v_local_sums); - hvx_vmem(local_delta_b) = hvx_vec_mul_f32_f32(diff, hvx_vec_splat_f32(beta_val)); - - gdn_add_scaled_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, - local_k, local_delta_b, local_q, S_v, local_sums); - - HVX_Vector res_attn = hvx_vec_mul_f32_f32(hvx_vmem(local_sums), hvx_vec_splat_f32(scale)); - hvx_vec_store_u(attn_data + j, 8 * sizeof(float), res_attn); - } - for (; j + 4 <= S_v; j += 4) { - float * row0 = s_work_curr + (uint64_t) (j + 0) * S_v; - float * row1 = s_work_curr + (uint64_t) (j + 1) * S_v; - float * row2 = s_work_curr + (uint64_t) (j + 2) * S_v; - float * row3 = s_work_curr + (uint64_t) (j + 3) * S_v; - gdn_mul_scalar_dot4_f32(row0, row1, row2, row3, gate, local_k, S_v, local_sums); - - float local_delta_b[32] __attribute__((aligned(128))); - HVX_Vector vv_t = hvx_vmemu(v_t + j); - HVX_Vector v_local_sums = hvx_vmem(local_sums); - HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, v_local_sums); - hvx_vmem(local_delta_b) = hvx_vec_mul_f32_f32(diff, hvx_vec_splat_f32(beta_val)); - - gdn_add_scaled_dot4_f32(row0, row1, row2, row3, local_k, local_delta_b, local_q, S_v, local_sums); - - HVX_Vector res_attn = hvx_vec_mul_f32_f32(hvx_vmem(local_sums), hvx_vec_splat_f32(scale)); - hvx_vec_store_u(attn_data + j, 4 * sizeof(float), res_attn); - } - HVX_Vector vscale_splat = hvx_vec_splat_f32(scale); - for (; j < S_v; ++j) { - float * row = s_work_curr + (uint64_t) j * S_v; - HVX_Vector vsum = gdn_mul_scalar_dot_f32(row, gate, local_k, S_v); - HVX_Vector vv_t = hvx_vec_splat_f32(v_t[j]); - HVX_Vector vdj = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv_t, vsum), hvx_vec_splat_f32(beta_val)); - HVX_Vector vres = gdn_add_scaled_dot_f32(row, local_k, vdj, local_q, S_v); - attn_data[j] = hvx_vec_get_f32(hvx_vec_mul_f32_f32(vres, vscale_splat)); - } + gdn_step_scalar_f32(s_work_curr, attn_data, q_t, k_t, v_t, g_t, beta_val, scale, S_v); } if (K > 1) { - // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back. const int64_t target_slot = (int64_t) n_tokens - 1 - (int64_t) t; - if (target_slot >= 0 && target_slot < (int64_t) K) { + if (target_slot > 0 && target_slot < (int64_t) K) { float * curr_state_o = state_out_base + (uint64_t) target_slot * state_size_per_snap + ((uint64_t) iv3 * H + iv1) * S_v * S_v; - if (curr_state_o != s_out) { - hvx_copy_f32_uu((uint8_t *) curr_state_o, (const uint8_t *) s_work_curr, S_v * S_v); - } + hvx_copy_f32_uu((uint8_t *) curr_state_o, (const uint8_t *) s_work_curr, S_v * S_v); } } attn_data += (uint64_t) S_v * H; } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); // Push real write-back - dma_queue_push(dma, dma_make_ptr(s_out, s_work_curr), + dma_queue_push(dma_q, dma_make_data(s_out, s_work_curr), S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), S_v); // Prefetch next block (if any) - if (ir_prefetch < total_rows) { - const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); - const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); - const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; + if (ir_prefetch < row_end) { + const uint32_t piv1 = fastmodulo(ir_prefetch, H, fd_H); + const uint32_t piv3 = fastdiv(ir_prefetch, fd_H); + dma_addr_t ps_in = state->data + ((uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v) * sizeof(float); - dma_queue_push(dma, dma_make_ptr(s_work[spad_idx], ps_in), + dma_queue_push(dma_q, dma_make_data(s_work[spad_idx], ps_in), S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), S_v); @@ -827,13 +889,13 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo curr_spad_idx ^= 1; } - dma_queue_flush(dma); + dma_queue_flush(dma_q); } - static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, void * data) { struct htp_gdn_context * gctx = (struct htp_gdn_context *) data; struct htp_ops_context * octx = gctx->octx; + const struct htp_gdn_kernel_params * kparams = gctx->kparams; const struct htp_tensor * q = octx->src[0]; const struct htp_tensor * k = octx->src[1]; @@ -843,62 +905,51 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo const struct htp_tensor * state = octx->src[5]; const struct htp_tensor * dst = octx->dst; - const uint32_t S_v = v->ne[0]; - const uint32_t H = v->ne[1]; - const uint32_t n_seqs = v->ne[3]; + const uint32_t S_v = kparams->S_v; + const uint32_t H = kparams->H; + const uint32_t n_seqs = kparams->n_seqs; + const uint32_t row_end = gctx->row_start + gctx->nrows; - const uint32_t total_rows = H * n_seqs; - if (ith >= total_rows) { + if (ith >= gctx->nrows) { return; } - const uint32_t rq3 = n_seqs / q->ne[3]; - const uint32_t rk3 = n_seqs / k->ne[3]; - const float scale = 1.0f / sqrtf((float) S_v); + const struct htp_tensor * dst_cache = octx->dsts[1]; + const float scale = kparams->scale; + float * dst_base = (float *) (uintptr_t) dst->data; - float * dst_base = (float *) (uintptr_t) dst->data; - float * state_out_base = dst_base + (uint64_t) S_v * H * n_seqs; - const float * state_in_base = (const float *) (uintptr_t) state->data; - - const bool kda = (g->ne[0] == S_v); - float local_gate[HTP_GDN_MAX_SV] __attribute__((aligned(128))); - float local_q[HTP_GDN_MAX_SV] __attribute__((aligned(128))); - float local_k[HTP_GDN_MAX_SV] __attribute__((aligned(128))); - float local_sums[32] __attribute__((aligned(128))); - - dma_queue * dma = octx->ctx->dma[ith]; - size_t state_aligned = (size_t) S_v * S_v * sizeof(float); - state_aligned = (state_aligned + 127) & ~(size_t)127; + dma_queue * dma_q = octx->ctx->dma[ith]; + const struct htp_gdn_vtcm_layout * layout = &gctx->layout; float * s_work[2]; - s_work[0] = (float *) (gctx->vtcm_base + gctx->vtcm_per_thread * ith); - s_work[1] = s_work[0] + state_aligned / sizeof(float); + s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith); + s_work[1] = s_work[0] + layout->state_aligned / sizeof(float); - struct fastdiv_values fd_H = init_fastdiv_values(H); - struct fastdiv_values fd_q1 = init_fastdiv_values(q->ne[1]); - struct fastdiv_values fd_k1 = init_fastdiv_values(k->ne[1]); - struct fastdiv_values fd_rq3 = init_fastdiv_values(rq3); - struct fastdiv_values fd_rk3 = init_fastdiv_values(rk3); + const struct fastdiv_values * fd_H = &kparams->div_H; + const struct fastdiv_values * fd_q1 = &kparams->div_q1; + const struct fastdiv_values * fd_k1 = &kparams->div_k1; + const struct fastdiv_values * fd_rq3 = &kparams->div_rq3; + const struct fastdiv_values * fd_rk3 = &kparams->div_rk3; - const uint64_t state_seq_stride = state->nb[3] / sizeof(float); + const uint32_t state_seq_stride = kparams->state_seq_stride; + const dma_addr_t state_out_dma_base = dst_cache ? dst_cache->data : (dst->data + S_v * H * n_seqs * sizeof(float)); - uint32_t ir_prefetch = ith; + uint32_t ir_prefetch = gctx->row_start + ith; int spad_idx = 0; // Prefetch preamble (up to 2 steps) - for (int k = 0; k < 2 && ir_prefetch < total_rows; k++) { - const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); - const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); - const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; - // final state lands in snapshot slot 0 (most-recent-first ordering) - float * ps_out = state_out_base + ((uint64_t) piv3 * H + piv1) * S_v * S_v; + for (int step = 0; step < 2 && ir_prefetch < row_end; step++) { + const uint32_t piv1 = fastmodulo(ir_prefetch, H, fd_H); + const uint32_t piv3 = fastdiv(ir_prefetch, fd_H); + dma_addr_t ps_in = state->data + ((uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v) * sizeof(float); + dma_addr_t ps_out = state_out_dma_base + ((uint64_t) piv3 * H + piv1) * S_v * S_v * sizeof(float); // Push dummy write-back - dma_queue_push(dma, dma_make_ptr(ps_out, s_work[spad_idx]), + dma_queue_push(dma_q, dma_make_data(ps_out, s_work[spad_idx]), S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), 0); // Push fetch - dma_queue_push(dma, dma_make_ptr(s_work[spad_idx], ps_in), + dma_queue_push(dma_q, dma_make_data(s_work[spad_idx], ps_in), S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), S_v); @@ -906,24 +957,24 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + int curr_spad_idx = 0; - for (uint32_t ir = ith; ir < total_rows; ir += nth) { - dma_queue_pop(dma); - dma_queue_pop(dma); + for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); float * s_work_curr = s_work[curr_spad_idx]; - const uint32_t iv1 = fastmodulo(ir, H, &fd_H); - const uint32_t iv3 = fastdiv(ir, &fd_H); - - const uint32_t iq1 = fastmodulo(iv1, q->ne[1], &fd_q1); - const uint32_t ik1 = fastmodulo(iv1, k->ne[1], &fd_k1); - const uint32_t iq3 = fastdiv(iv3, &fd_rq3); - const uint32_t ik3 = fastdiv(iv3, &fd_rk3); + const uint32_t iv1 = fastmodulo(ir, H, fd_H); + const uint32_t iv3 = fastdiv(ir, fd_H); - // final state lands in snapshot slot 0 (most-recent-first ordering) - float * s_out = state_out_base + ((uint64_t) iv3 * H + iv1) * S_v * S_v; + const uint32_t iq1 = fastmodulo(iv1, q->ne[1], fd_q1); + const uint32_t ik1 = fastmodulo(iv1, k->ne[1], fd_k1); + const uint32_t iq3 = fastdiv(iv3, fd_rq3); + const uint32_t ik3 = fastdiv(iv3, fd_rk3); + dma_addr_t s_out = state_out_dma_base + ((uint64_t) iv3 * H + iv1) * S_v * S_v * sizeof(float); float * attn_data = dst_base + ((uint64_t) iv3 * H + iv1) * S_v; const float * q_t = (const float *) ((const uint8_t *) (uintptr_t) q->data + @@ -937,132 +988,26 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo const float beta_val = *(const float *) ((const uint8_t *) (uintptr_t) beta->data + (uint64_t) iv3 * beta->nb[3] + (uint64_t) iv1 * beta->nb[1]); - hvx_copy_f32_au((uint8_t *) local_q, (const uint8_t *) q_t, S_v); - hvx_copy_f32_au((uint8_t *) local_k, (const uint8_t *) k_t, S_v); - - if (kda) { - hvx_exp_f32((uint8_t *) local_gate, (const uint8_t *) g_t, S_v, false); - - uint32_t j = 0; - for (; j + 8 <= S_v; j += 8) { - float * row0 = s_work_curr + (uint64_t) (j + 0) * S_v; - float * row1 = s_work_curr + (uint64_t) (j + 1) * S_v; - float * row2 = s_work_curr + (uint64_t) (j + 2) * S_v; - float * row3 = s_work_curr + (uint64_t) (j + 3) * S_v; - float * row4 = s_work_curr + (uint64_t) (j + 4) * S_v; - float * row5 = s_work_curr + (uint64_t) (j + 5) * S_v; - float * row6 = s_work_curr + (uint64_t) (j + 6) * S_v; - float * row7 = s_work_curr + (uint64_t) (j + 7) * S_v; - gdn_mul_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, - local_gate, local_k, S_v, local_sums); - - float local_delta_b[32] __attribute__((aligned(128))); - HVX_Vector vv_t = hvx_vmemu(v_t + j); - HVX_Vector v_local_sums = hvx_vmem(local_sums); - HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, v_local_sums); - hvx_vmem(local_delta_b) = hvx_vec_mul_f32_f32(diff, hvx_vec_splat_f32(beta_val)); - - gdn_add_scaled_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, - local_k, local_delta_b, local_q, S_v, local_sums); - - HVX_Vector res_attn = hvx_vec_mul_f32_f32(hvx_vmem(local_sums), hvx_vec_splat_f32(scale)); - hvx_vec_store_u(attn_data + j, 8 * sizeof(float), res_attn); - } - for (; j + 4 <= S_v; j += 4) { - float * row0 = s_work_curr + (uint64_t) (j + 0) * S_v; - float * row1 = s_work_curr + (uint64_t) (j + 1) * S_v; - float * row2 = s_work_curr + (uint64_t) (j + 2) * S_v; - float * row3 = s_work_curr + (uint64_t) (j + 3) * S_v; - gdn_mul_dot4_f32(row0, row1, row2, row3, local_gate, local_k, S_v, local_sums); - - float local_delta_b[32] __attribute__((aligned(128))); - HVX_Vector vv_t = hvx_vmemu(v_t + j); - HVX_Vector v_local_sums = hvx_vmem(local_sums); - HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, v_local_sums); - hvx_vmem(local_delta_b) = hvx_vec_mul_f32_f32(diff, hvx_vec_splat_f32(beta_val)); - - gdn_add_scaled_dot4_f32(row0, row1, row2, row3, local_k, local_delta_b, local_q, S_v, local_sums); - - HVX_Vector res_attn = hvx_vec_mul_f32_f32(hvx_vmem(local_sums), hvx_vec_splat_f32(scale)); - hvx_vec_store_u(attn_data + j, 4 * sizeof(float), res_attn); - } - HVX_Vector vscale_splat = hvx_vec_splat_f32(scale); - for (; j < S_v; ++j) { - float * row = s_work_curr + (uint64_t) j * S_v; - HVX_Vector vsum = gdn_mul_dot_f32(row, local_gate, local_k, S_v); - HVX_Vector vv_t = hvx_vec_splat_f32(v_t[j]); - HVX_Vector vdj = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv_t, vsum), hvx_vec_splat_f32(beta_val)); - HVX_Vector vres = gdn_add_scaled_dot_f32(row, local_k, vdj, local_q, S_v); - attn_data[j] = hvx_vec_get_f32(hvx_vec_mul_f32_f32(vres, vscale_splat)); - } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + if (kparams->kda) { + gdn_step_kda_f32(s_work_curr, attn_data, q_t, k_t, v_t, g_t, beta_val, scale, S_v); } else { - const float gate = expf(g_t[0]); - uint32_t j = 0; - for (; j + 8 <= S_v; j += 8) { - float * row0 = s_work_curr + (uint64_t) (j + 0) * S_v; - float * row1 = s_work_curr + (uint64_t) (j + 1) * S_v; - float * row2 = s_work_curr + (uint64_t) (j + 2) * S_v; - float * row3 = s_work_curr + (uint64_t) (j + 3) * S_v; - float * row4 = s_work_curr + (uint64_t) (j + 4) * S_v; - float * row5 = s_work_curr + (uint64_t) (j + 5) * S_v; - float * row6 = s_work_curr + (uint64_t) (j + 6) * S_v; - float * row7 = s_work_curr + (uint64_t) (j + 7) * S_v; - gdn_mul_scalar_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, - gate, local_k, S_v, local_sums); - - float local_delta_b[32] __attribute__((aligned(128))); - HVX_Vector vv_t = hvx_vmemu(v_t + j); - HVX_Vector v_local_sums = hvx_vmem(local_sums); - HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, v_local_sums); - hvx_vmem(local_delta_b) = hvx_vec_mul_f32_f32(diff, hvx_vec_splat_f32(beta_val)); - - gdn_add_scaled_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, - local_k, local_delta_b, local_q, S_v, local_sums); - - HVX_Vector res_attn = hvx_vec_mul_f32_f32(hvx_vmem(local_sums), hvx_vec_splat_f32(scale)); - hvx_vec_store_u(attn_data + j, 8 * sizeof(float), res_attn); - } - for (; j + 4 <= S_v; j += 4) { - float * row0 = s_work_curr + (uint64_t) (j + 0) * S_v; - float * row1 = s_work_curr + (uint64_t) (j + 1) * S_v; - float * row2 = s_work_curr + (uint64_t) (j + 2) * S_v; - float * row3 = s_work_curr + (uint64_t) (j + 3) * S_v; - gdn_mul_scalar_dot4_f32(row0, row1, row2, row3, gate, local_k, S_v, local_sums); - - float local_delta_b[32] __attribute__((aligned(128))); - HVX_Vector vv_t = hvx_vmemu(v_t + j); - HVX_Vector v_local_sums = hvx_vmem(local_sums); - HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, v_local_sums); - hvx_vmem(local_delta_b) = hvx_vec_mul_f32_f32(diff, hvx_vec_splat_f32(beta_val)); - - gdn_add_scaled_dot4_f32(row0, row1, row2, row3, local_k, local_delta_b, local_q, S_v, local_sums); - - HVX_Vector res_attn = hvx_vec_mul_f32_f32(hvx_vmem(local_sums), hvx_vec_splat_f32(scale)); - hvx_vec_store_u(attn_data + j, 4 * sizeof(float), res_attn); - } - HVX_Vector vscale_splat = hvx_vec_splat_f32(scale); - for (; j < S_v; ++j) { - float * row = s_work_curr + (uint64_t) j * S_v; - HVX_Vector vsum = gdn_mul_scalar_dot_f32(row, gate, local_k, S_v); - HVX_Vector vv_t = hvx_vec_splat_f32(v_t[j]); - HVX_Vector vdj = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv_t, vsum), hvx_vec_splat_f32(beta_val)); - HVX_Vector vres = gdn_add_scaled_dot_f32(row, local_k, vdj, local_q, S_v); - attn_data[j] = hvx_vec_get_f32(hvx_vec_mul_f32_f32(vres, vscale_splat)); - } + gdn_step_scalar_f32(s_work_curr, attn_data, q_t, k_t, v_t, g_t, beta_val, scale, S_v); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); // Push real write-back - dma_queue_push(dma, dma_make_ptr(s_out, s_work_curr), + dma_queue_push(dma_q, dma_make_data(s_out, s_work_curr), S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), S_v); // Prefetch next block (if any) - if (ir_prefetch < total_rows) { - const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); - const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); - const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; + if (ir_prefetch < row_end) { + const uint32_t piv1 = fastmodulo(ir_prefetch, H, fd_H); + const uint32_t piv3 = fastdiv(ir_prefetch, fd_H); + dma_addr_t ps_in = state->data + ((uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v) * sizeof(float); - dma_queue_push(dma, dma_make_ptr(s_work[spad_idx], ps_in), + dma_queue_push(dma_q, dma_make_data(s_work[spad_idx], ps_in), S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), S_v); @@ -1072,9 +1017,1257 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo curr_spad_idx ^= 1; } - dma_queue_flush(dma); + dma_queue_flush(dma_q); +} + +struct htp_gdn_hmx_gemm_task { + const __fp16 * row_tiles; + const __fp16 * col_tiles; + __fp16 * out_tiles; + uint32_t n_row_tiles; + uint32_t n_col_tiles; + uint32_t n_dot_tiles; + uint32_t dot_stride; + uint8_t * hmx_scales; +}; + +static void htp_gdn_hmx_gemm_worker(void * data) { + struct htp_gdn_hmx_gemm_task * task = (struct htp_gdn_hmx_gemm_task *) data; + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)task->hmx_scales)); + + const size_t dot_stride = task->dot_stride; + for (uint32_t r = 0; r < task->n_row_tiles; ++r) { + const __fp16 * r_tiles = task->row_tiles + r * dot_stride; + const __fp16 * c_tiles = task->col_tiles; + __fp16 * o_tile = task->out_tiles + r * task->n_col_tiles * HMX_FP16_TILE_N_ELMS; + + for (uint32_t c = 0; c < task->n_col_tiles; ++c) { + hmx_fa_qk_dot_tile(r_tiles, c_tiles, o_tile, task->n_dot_tiles); + c_tiles += dot_stride; + o_tile += HMX_FP16_TILE_N_ELMS; + } + } +} + +static inline void htp_gdn_push_hmx_gemm_task( + hmx_queue_t q, + struct htp_gdn_hmx_gemm_task * task, + const __fp16 * row_tiles, + const __fp16 * col_tiles, + __fp16 * out_tiles, + uint32_t n_row_tiles, + uint32_t n_col_tiles, + uint32_t n_dot_tiles, + uint8_t * scales +) { + task->row_tiles = row_tiles; + task->col_tiles = col_tiles; + task->out_tiles = out_tiles; + task->n_row_tiles = n_row_tiles; + task->n_col_tiles = n_col_tiles; + task->n_dot_tiles = n_dot_tiles; + task->dot_stride = n_dot_tiles * HMX_FP16_TILE_N_ELMS; + task->hmx_scales = scales; + + hmx_queue_push(q, hmx_queue_make_desc(htp_gdn_hmx_gemm_worker, task)); +} + +static inline void gdn_unpack_64x64_tiles_to_vectors( + HVX_Vector * restrict rows, + const __fp16 * restrict tiles +) { + const HVX_Vector * t00 = (const HVX_Vector *) (tiles + 0 * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t01 = (const HVX_Vector *) (tiles + 1 * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t10 = (const HVX_Vector *) (tiles + 2 * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t11 = (const HVX_Vector *) (tiles + 3 * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp0 = Q6_W_vdeal_VVR(t01[r], t00[r], -2); + rows[2 * r + 0] = Q6_V_lo_W(vp0); + rows[2 * r + 1] = Q6_V_hi_W(vp0); + + HVX_VectorPair vp1 = Q6_W_vdeal_VVR(t11[r], t10[r], -2); + rows[32 + 2 * r + 0] = Q6_V_lo_W(vp1); + rows[32 + 2 * r + 1] = Q6_V_hi_W(vp1); + } +} + +static inline void gdn_pack_64x64_vectors_to_tiles( + __fp16 * restrict tiles, + const HVX_Vector * restrict rows +) { + HVX_Vector * t00 = (HVX_Vector *) (tiles + 0 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * t01 = (HVX_Vector *) (tiles + 1 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * t10 = (HVX_Vector *) (tiles + 2 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * t11 = (HVX_Vector *) (tiles + 3 * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp0 = Q6_W_vshuff_VVR(rows[2 * r + 1], rows[2 * r + 0], -2); + t00[r] = Q6_V_lo_W(vp0); + t01[r] = Q6_V_hi_W(vp0); + + HVX_VectorPair vp1 = Q6_W_vshuff_VVR(rows[32 + 2 * r + 1], rows[32 + 2 * r + 0], -2); + t10[r] = Q6_V_lo_W(vp1); + t11[r] = Q6_V_hi_W(vp1); + } +} + +static inline void gdn_unpack_64xS_tiles_to_f32( + float * restrict dst_f32, + const __fp16 * restrict tiles, + uint32_t S_v +) { + const uint32_t n_col_tiles = S_v / 32; + for (uint32_t r0 = 0; r0 < 2; ++r0) { + for (uint32_t d = 0; d < S_v / 64; ++d) { + const HVX_Vector * t0 = (const HVX_Vector *) (tiles + (r0 * n_col_tiles + 2 * d + 0) * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t1 = (const HVX_Vector *) (tiles + (r0 * n_col_tiles + 2 * d + 1) * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp01 = Q6_W_vdeal_VVR(t1[r], t0[r], -2); + HVX_VectorPair p0 = hvx_vec_f16_to_f32(Q6_V_lo_W(vp01)); + HVX_VectorPair p1 = hvx_vec_f16_to_f32(Q6_V_hi_W(vp01)); + + float * out0 = dst_f32 + (r0 * 32 + 2 * r + 0) * S_v + d * 64; + float * out1 = dst_f32 + (r0 * 32 + 2 * r + 1) * S_v + d * 64; + + hvx_vmem(out0 + 0) = Q6_V_lo_W(p0); + hvx_vmem(out0 + 32) = Q6_V_hi_W(p0); + hvx_vmem(out1 + 0) = Q6_V_lo_W(p1); + hvx_vmem(out1 + 32) = Q6_V_hi_W(p1); + } + } + } +} + +static inline void gdn_unpack_64xS_tiles_to_f16( + __fp16 * restrict dst_f16, + const __fp16 * restrict tiles, + uint32_t S_v +) { + const uint32_t n_col_tiles = S_v / 32; + for (uint32_t r0 = 0; r0 < 2; ++r0) { + for (uint32_t d = 0; d < S_v / 64; ++d) { + const HVX_Vector * t0 = (const HVX_Vector *) (tiles + (r0 * n_col_tiles + 2 * d + 0) * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t1 = (const HVX_Vector *) (tiles + (r0 * n_col_tiles + 2 * d + 1) * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp01 = Q6_W_vdeal_VVR(t1[r], t0[r], -2); + __fp16 * out0 = dst_f16 + (r0 * 32 + 2 * r + 0) * S_v + d * 64; + __fp16 * out1 = dst_f16 + (r0 * 32 + 2 * r + 1) * S_v + d * 64; + + hvx_vmem(out0) = Q6_V_lo_W(vp01); + hvx_vmem(out1) = Q6_V_hi_W(vp01); + } + } + } +} + +static inline void gdn_unpack_SxS_tiles_to_f32( + float * restrict dst_f32, + const __fp16 * restrict tiles, + uint32_t S_v +) { + const uint32_t n_tiles = S_v / 32; + for (uint32_t r0 = 0; r0 < n_tiles; ++r0) { + for (uint32_t d = 0; d < S_v / 64; ++d) { + const HVX_Vector * t0 = (const HVX_Vector *) (tiles + (r0 * n_tiles + 2 * d + 0) * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t1 = (const HVX_Vector *) (tiles + (r0 * n_tiles + 2 * d + 1) * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp01 = Q6_W_vdeal_VVR(t1[r], t0[r], -2); + HVX_VectorPair p0 = hvx_vec_f16_to_f32(Q6_V_lo_W(vp01)); + HVX_VectorPair p1 = hvx_vec_f16_to_f32(Q6_V_hi_W(vp01)); + + float * out0 = dst_f32 + (r0 * 32 + 2 * r + 0) * S_v + d * 64; + float * out1 = dst_f32 + (r0 * 32 + 2 * r + 1) * S_v + d * 64; + + hvx_vmem(out0 + 0) = Q6_V_lo_W(p0); + hvx_vmem(out0 + 32) = Q6_V_hi_W(p0); + hvx_vmem(out1 + 0) = Q6_V_lo_W(p1); + hvx_vmem(out1 + 32) = Q6_V_hi_W(p1); + } + } + } +} + +static inline void gdn_f32_to_hmx_row_tiles_and_f16( + __fp16 * restrict dst_tiles, + __fp16 * restrict dst_prime_tiles, + __fp16 * restrict dst_f16, + const float * restrict src, + const __fp16 * restrict scale_per_row, + uint32_t n_rows, + uint32_t n_cols +) { + const uint32_t n_col_tiles = n_cols / 32; + const uint32_t * scale_pairs = (const uint32_t *) scale_per_row; + + for (uint32_t r = 0; r < n_rows; r += 2) { + uint32_t r0 = r / 32; + uint32_t r1 = (r % 32) / 2; + const float * p0 = src + (r + 0) * n_cols; + const float * p1 = src + (r + 1) * n_cols; + + HVX_Vector v_scale; + if (dst_prime_tiles) { + uint32_t scale_pair = scale_pairs ? scale_pairs[r / 2] : 0x3c003c00; + v_scale = Q6_V_vsplat_R(scale_pair); + } + + for (uint32_t c = 0; c < n_col_tiles; c += 2) { + HVX_Vector v0_0 = hvx_vmem(p0 + (c + 0) * 32); + HVX_Vector v1_0 = hvx_vmem(p1 + (c + 0) * 32); + HVX_Vector v0_1 = hvx_vmem(p0 + (c + 1) * 32); + HVX_Vector v1_1 = hvx_vmem(p1 + (c + 1) * 32); + + HVX_Vector vh0 = hvx_vec_f32_to_f16_shuff(v0_0, v1_0); + HVX_Vector vh1 = hvx_vec_f32_to_f16_shuff(v0_1, v1_1); + __fp16 * tile0 = dst_tiles + (r0 * n_col_tiles + c + 0) * HMX_FP16_TILE_N_ELMS; + __fp16 * tile1 = dst_tiles + (r0 * n_col_tiles + c + 1) * HMX_FP16_TILE_N_ELMS; + ((HVX_Vector *) tile0)[r1] = vh0; + ((HVX_Vector *) tile1)[r1] = vh1; + + if (dst_prime_tiles) { + HVX_Vector vh0_s = hvx_vec_mul_f16_f16(vh0, v_scale); + HVX_Vector vh1_s = hvx_vec_mul_f16_f16(vh1, v_scale); + __fp16 * tile0_s = dst_prime_tiles + (r0 * n_col_tiles + c + 0) * HMX_FP16_TILE_N_ELMS; + __fp16 * tile1_s = dst_prime_tiles + (r0 * n_col_tiles + c + 1) * HMX_FP16_TILE_N_ELMS; + ((HVX_Vector *) tile0_s)[r1] = vh0_s; + ((HVX_Vector *) tile1_s)[r1] = vh1_s; + } + + if (dst_f16) { + HVX_VectorPair vp01 = Q6_W_vdeal_VVR(vh1, vh0, -2); + hvx_vmem(dst_f16 + (r + 0) * n_cols + c * 32) = Q6_V_lo_W(vp01); + hvx_vmem(dst_f16 + (r + 1) * n_cols + c * 32) = Q6_V_hi_W(vp01); + } + } + } +} + +static inline void hvx_transpose_32x32_words(HVX_Vector * restrict m, HVX_Vector * restrict tmp) { + for (int i = 0; i < 16; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[2*i + 1], m[2*i], -4); + tmp[2*i + 0] = Q6_V_lo_W(p); + tmp[2*i + 1] = Q6_V_hi_W(p); + } + + for (int b = 0; b < 32; b += 4) { + HVX_VectorPair p0 = Q6_W_vshuff_VVR(tmp[b + 2], tmp[b + 0], -8); + HVX_VectorPair p1 = Q6_W_vshuff_VVR(tmp[b + 3], tmp[b + 1], -8); + m[b + 0] = Q6_V_lo_W(p0); m[b + 1] = Q6_V_hi_W(p0); + m[b + 2] = Q6_V_lo_W(p1); m[b + 3] = Q6_V_hi_W(p1); + } + + for (int b = 0; b < 32; b += 8) { + for (int i = 0; i < 4; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[b + i + 4], m[b + i], -16); + tmp[b + 2*i + 0] = Q6_V_lo_W(p); + tmp[b + 2*i + 1] = Q6_V_hi_W(p); + } + } + + for (int b = 0; b < 32; b += 16) { + for (int i = 0; i < 8; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(tmp[b + i + 8], tmp[b + i], -32); + m[b + 2*i + 0] = Q6_V_lo_W(p); + m[b + 2*i + 1] = Q6_V_hi_W(p); + } + } + + for (int i = 0; i < 16; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[i + 16], m[i], -64); + tmp[2 * i + 0] = Q6_V_lo_W(p); + tmp[2 * i + 1] = Q6_V_hi_W(p); + } + + for (int i = 0; i < 32; ++i) { + m[i] = tmp[i]; + } +} + +static inline void gdn_pack_d_t_row_tiles( + __fp16 * restrict dst_tiles, + const __fp16 * restrict src_d, + uint32_t S_v, + HVX_Vector * restrict m, + HVX_Vector * restrict tmp +) { + for (uint32_t col_half = 0; col_half < S_v / 64; ++col_half) { + uint32_t r0_base = col_half * 2; + for (uint32_t c0 = 0; c0 < 2; ++c0) { + for (uint32_t s_local = 0; s_local < 32; ++s_local) { + uint32_t s = c0 * 32 + s_local; + m[s_local] = hvx_vmem(src_d + s * S_v + col_half * 64); + } + + hvx_transpose_32x32_words(m, tmp); + + uint32_t tile0_idx = (r0_base + 0) * 2 + c0; + uint32_t tile1_idx = (r0_base + 1) * 2 + c0; + HVX_Vector * t0 = (HVX_Vector *)(dst_tiles + tile0_idx * HMX_FP16_TILE_N_ELMS); + HVX_Vector * t1 = (HVX_Vector *)(dst_tiles + tile1_idx * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + t0[r] = m[r]; + t1[r] = m[16 + r]; + } + } + } +} + +static __attribute__((noinline)) void gdn_build_inv_l_blocks( + __fp16 * restrict inv_row_tiles, + const HVX_Vector * restrict rows_kk, + const __fp16 * restrict decay_m, + const float * restrict beta, + __fp16 * restrict l10_tile, + __fp16 * restrict neg_a11_tile +) { + const HVX_Vector v_one_f16 = hvx_vec_splat_f16(1.0f); + const HVX_VectorPred q_mask64 = Q6_Q_vsetq2_R(64); + + uint16_t beta_u16[64] __attribute__((aligned(128))); + uint16_t l00[32][32] __attribute__((aligned(128))); + uint16_t l11[32][32] __attribute__((aligned(128))); + + HVX_Vector * restrict p_l00 = (HVX_Vector *) l00; + HVX_Vector * restrict p_l11 = (HVX_Vector *) l11; + HVX_Vector * restrict p_l10_tile = (HVX_Vector *) l10_tile; + + HVX_Vector * restrict tile00 = (HVX_Vector *) (inv_row_tiles + 0 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * restrict tile01 = (HVX_Vector *) (inv_row_tiles + 1 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * restrict tile11 = (HVX_Vector *) (inv_row_tiles + 3 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * restrict p_neg_a11 = (HVX_Vector *) neg_a11_tile; + + hvx_vmem(beta_u16) = hvx_vec_f32_to_f16(hvx_vmem(beta + 0), hvx_vmem(beta + 32)); + + for (uint32_t r = 0; r < 16; ++r) { + tile01[r] = Q6_V_vzero(); + } + + for (uint32_t r = 0; r < 16; ++r) { + uint32_t t0 = 2 * r; + uint32_t t1 = t0 + 1; + + HVX_Vector v_d0 = hvx_vmem(decay_m + t0 * 64); + HVX_Vector v_d1 = hvx_vmem(decay_m + t1 * 64); + HVX_Vector v_b0 = Q6_Vh_vsplat_R(beta_u16[t0]); + HVX_Vector v_b1 = Q6_Vh_vsplat_R(beta_u16[t1]); + + HVX_Vector r0 = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(rows_kk[t0], v_d0), v_b0); + HVX_Vector r1 = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(rows_kk[t1], v_d1), v_b1); + + p_l00[r] = Q6_V_vmux_QVV(q_mask64, r0, Q6_V_vror_VR(r1, 64)); + } + + for (uint32_t r = 0; r < 16; ++r) { + uint32_t t0 = 32 + 2 * r; + uint32_t t1 = t0 + 1; + + HVX_Vector v_d0 = hvx_vmem(decay_m + t0 * 64); + HVX_Vector v_d1 = hvx_vmem(decay_m + t1 * 64); + HVX_Vector v_b0 = Q6_Vh_vsplat_R(beta_u16[t0]); + HVX_Vector v_b1 = Q6_Vh_vsplat_R(beta_u16[t1]); + + HVX_Vector r0 = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(rows_kk[t0], v_d0), v_b0); + HVX_Vector r1 = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(rows_kk[t1], v_d1), v_b1); + + HVX_VectorPair vp_l10 = Q6_W_vshuff_VVR(r1, r0, -2); + p_l10_tile[r] = Q6_V_lo_W(vp_l10); + p_l11[r] = Q6_V_vmux_QVV(q_mask64, Q6_V_vror_VR(r0, 64), r1); + } + + HVX_Vector a_rows[32]; + for (uint32_t t = 0; t < 32; ++t) { + HVX_Vector v_inv = Q6_V_vzero(); + for (uint32_t k = 0; k < t; ++k) { + HVX_Vector v_lk = Q6_Vh_vsplat_R(l00[t][k]); + v_inv = hvx_vec_sub_f16_f16(v_inv, hvx_vec_mul_f16_f16(v_lk, a_rows[k])); + } + HVX_VectorPred q_diag = (t == 0) ? Q6_Q_vsetq2_R(2) : Q6_Q_and_QQn(Q6_Q_vsetq2_R(2 * (t + 1)), Q6_Q_vsetq2_R(2 * t)); + a_rows[t] = Q6_V_vand_QV(q_mask64, Q6_V_vmux_QVV(q_diag, v_one_f16, v_inv)); + } + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp = Q6_W_vshuff_VVR(a_rows[2 * r + 1], a_rows[2 * r + 0], -2); + tile00[r] = Q6_V_lo_W(vp); + } + + for (uint32_t t = 0; t < 32; ++t) { + HVX_Vector v_inv = Q6_V_vzero(); + for (uint32_t k = 0; k < t; ++k) { + HVX_Vector v_lk = Q6_Vh_vsplat_R(l11[t][k]); + v_inv = hvx_vec_sub_f16_f16(v_inv, hvx_vec_mul_f16_f16(v_lk, a_rows[k])); + } + HVX_VectorPred q_diag = (t == 0) ? Q6_Q_vsetq2_R(2) : Q6_Q_and_QQn(Q6_Q_vsetq2_R(2 * (t + 1)), Q6_Q_vsetq2_R(2 * t)); + a_rows[t] = Q6_V_vand_QV(q_mask64, Q6_V_vmux_QVV(q_diag, v_one_f16, v_inv)); + } + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp = Q6_W_vshuff_VVR(a_rows[2 * r + 1], a_rows[2 * r + 0], -2); + tile11[r] = Q6_V_lo_W(vp); + + HVX_Vector n0 = hvx_vec_sub_f16_f16(Q6_V_vzero(), a_rows[2 * r + 0]); + HVX_Vector n1 = hvx_vec_sub_f16_f16(Q6_V_vzero(), a_rows[2 * r + 1]); + HVX_VectorPair vp_neg = Q6_W_vshuff_VVR(n1, n0, -2); + p_neg_a11[r] = Q6_V_lo_W(vp_neg); + } +} + + +static inline void gdn_dma_push_chunk_inputs( + dma_queue * dma_q, + float * vtcm_q, + float * vtcm_k, + float * vtcm_v, + const struct htp_tensor * q, + const struct htp_tensor * k, + const struct htp_tensor * v, + uint32_t iq3, uint32_t iq1, + uint32_t ik3, uint32_t ik1, + uint32_t iv3, uint32_t iv1, + uint32_t t_chunk, + uint32_t chunk_size, + uint32_t S_v +) { + const dma_addr_t q_dma = q->data + (uint64_t) iq3 * q->nb[3] + (uint64_t) t_chunk * q->nb[2] + (uint64_t) iq1 * q->nb[1]; + const dma_addr_t k_dma = k->data + (uint64_t) ik3 * k->nb[3] + (uint64_t) t_chunk * k->nb[2] + (uint64_t) ik1 * k->nb[1]; + const dma_addr_t v_dma = v->data + (uint64_t) iv3 * v->nb[3] + (uint64_t) t_chunk * v->nb[2] + (uint64_t) iv1 * v->nb[1]; + + dma_queue_push(dma_q, dma_make_data(vtcm_q, q_dma), S_v * sizeof(float), q->nb[2], S_v * sizeof(float), chunk_size); + dma_queue_push(dma_q, dma_make_data(vtcm_k, k_dma), S_v * sizeof(float), k->nb[2], S_v * sizeof(float), chunk_size); + dma_queue_push(dma_q, dma_make_data(vtcm_v, v_dma), S_v * sizeof(float), v->nb[2], S_v * sizeof(float), chunk_size); } +static inline void gdn_dma_push_chunk_gb( + dma_queue * dma_q, + float * vtcm_g_raw, + float * vtcm_b_raw, + const struct htp_tensor * g, + const struct htp_tensor * beta, + uint32_t iv3, + uint32_t iv1, + uint32_t t_chunk, + uint32_t chunk_size, + uint32_t n_batch +) { + const dma_addr_t g_dma = g->data + (uint64_t) iv3 * g->nb[3] + (uint64_t) t_chunk * g->nb[2] + (uint64_t) iv1 * g->nb[1]; + const dma_addr_t beta_dma = beta->data + (uint64_t) iv3 * beta->nb[3] + (uint64_t) t_chunk * beta->nb[2] + (uint64_t) iv1 * beta->nb[1]; + const uint32_t row_bytes = n_batch * sizeof(float); + + dma_queue_push(dma_q, dma_make_data(vtcm_g_raw, g_dma), row_bytes, g->nb[2], row_bytes, chunk_size); + dma_queue_push(dma_q, dma_make_data(vtcm_b_raw, beta_dma), row_bytes, beta->nb[2], row_bytes, chunk_size); +} + +static inline void gdn_pack_s_col_tiles( + __fp16 * restrict vtcm_s_col_tiles, + __fp16 * restrict vtcm_s_f16, + const float * restrict vtcm_s_state, + uint32_t S_v +) { + for (uint32_t j = 0; j < S_v; ++j) { + for (uint32_t i = 0; i < S_v; i += 64) { + HVX_Vector v0 = hvx_vmem(vtcm_s_state + j * S_v + i + 0); + HVX_Vector v1 = (i + 32 < S_v) ? hvx_vmem(vtcm_s_state + j * S_v + i + 32) : Q6_V_vzero(); + hvx_vmem(vtcm_s_f16 + j * S_v + i) = hvx_vec_f32_to_f16(v0, v1); + } + } + hmx_interleave_rows_to_tiles(vtcm_s_col_tiles, vtcm_s_f16, S_v, S_v, S_v, 0, S_v); +} + +struct htp_gdn_head_ptrs { + float * s_state; + __fp16 * s_f16; + __fp16 * s_col_tiles; + float * s_update_f32; + __fp16 * s_update_tiles; + + float * q_f32[2]; + float * k_f32[2]; + float * v_f32[2]; + float * g_f32[2]; + float * b_f32[2]; + float * o_f32[2]; + + float * v_inter_f32; + float * o_inter_f32; + float * o_intra_f32; + + __fp16 * k_f16; + __fp16 * v_prime_f16; + __fp16 * delta_f16; + __fp16 * d_f16; + + __fp16 * q_row_tiles; + __fp16 * q_prime_row_tiles; + __fp16 * k_row_tiles; + __fp16 * k_col_tiles; + __fp16 * k_prime_row_tiles; + __fp16 * k_col_tiles_64x128; + __fp16 * kk_tiles; + __fp16 * qk_tiles; + __fp16 * v_inter_tiles; + __fp16 * o_inter_tiles; + __fp16 * inv_row_tiles; + __fp16 * a_row_tiles; + __fp16 * v_prime_col_tiles; + __fp16 * delta_tiles; + __fp16 * delta_col_tiles; + __fp16 * o_intra_tiles; + __fp16 * d_row_tiles; + + __fp16 * gamma; + float * lambda_init; + __fp16 * lambda_init_f16; + __fp16 * decay_m; + __fp16 * decay_a; + + HVX_Vector * rows_kk; + HVX_Vector * rows_qk; + HVX_Vector * rows_inv; + HVX_Vector * rows_a; + + HVX_Vector * vtcm_m; + HVX_Vector * vtcm_tmp; + + uint32_t iv1; + uint32_t iv3; + uint32_t iq1; + uint32_t ik1; + uint32_t iq3; + uint32_t ik3; + dma_addr_t state_in_dma; + dma_addr_t state_out_dma; +}; + +static inline void gdn_init_head_ptrs( + struct htp_gdn_head_ptrs * head, + const struct htp_gdn_hmx_vtcm_layout * L, + uint8_t * vtcm_base, + uint32_t h, + uint32_t base_iv1, + uint32_t iv3, + const struct htp_tensor * q, + const struct htp_tensor * k, + const struct htp_tensor * v, + const struct htp_tensor * state, + const struct htp_tensor * dst, + const struct htp_tensor * dst_cache, + const struct htp_gdn_kernel_params * kparams, + uint32_t S_v, + uint32_t H, + uint32_t n_tokens, + uint32_t chunk_size +) { + const size_t dma_scalar_sz = hex_round_up(chunk_size * sizeof(float), 128); + const size_t decay_sz = 64 * 64 * sizeof(__fp16); + const size_t row_vecs_sz = 64 * 128; + + head->s_state = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_s_state + h * L->state_f32_bytes); + head->s_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_s_f16 + h * L->state_f16_bytes); + head->s_col_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_s_col_tiles + h * L->state_tiles_bytes); + head->s_update_f32 = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_s_update_f32 + h * L->state_f32_bytes); + head->s_update_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_s_update_tiles + h * L->state_tiles_bytes); + + head->q_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_q_f32[0] + h * L->dma_chunk_bytes); + head->q_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_q_f32[1] + h * L->dma_chunk_bytes) : head->q_f32[0]; + head->k_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_k_f32[0] + h * L->dma_chunk_bytes); + head->k_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_k_f32[1] + h * L->dma_chunk_bytes) : head->k_f32[0]; + head->v_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_v_f32[0] + h * L->dma_chunk_bytes); + head->v_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_v_f32[1] + h * L->dma_chunk_bytes) : head->v_f32[0]; + head->g_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_g_f32[0] + h * dma_scalar_sz); + head->g_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_g_f32[1] + h * dma_scalar_sz) : head->g_f32[0]; + head->b_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_b_f32[0] + h * dma_scalar_sz); + head->b_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_b_f32[1] + h * dma_scalar_sz) : head->b_f32[0]; + head->o_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_o_f32[0] + h * L->dma_chunk_bytes); + head->o_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_o_f32[1] + h * L->dma_chunk_bytes) : head->o_f32[0]; + + head->v_inter_f32 = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_v_inter_f32 + h * L->dma_chunk_bytes); + head->o_inter_f32 = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_o_inter_f32 + h * L->dma_chunk_bytes); + head->o_intra_f32 = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_o_intra_f32 + h * L->dma_chunk_bytes); + + head->k_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_k_f16 + h * L->act_f16_bytes); + head->v_prime_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_v_prime_f16 + h * L->act_f16_bytes); + head->delta_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_delta_f16 + h * L->act_f16_bytes); + head->d_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_d_f16 + h * L->act_f16_bytes); + + head->q_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_q_row_tiles + h * L->tile_64xSv_bytes); + head->q_prime_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_q_prime_row_tiles + h * L->tile_64xSv_bytes); + head->k_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_k_row_tiles + h * L->tile_64xSv_bytes); + head->k_col_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_k_col_tiles + h * L->tile_64xSv_bytes); + head->k_prime_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_k_prime_row_tiles + h * L->tile_64xSv_bytes); + head->k_col_tiles_64x128 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_k_col_tiles_64x128 + h * L->tile_64xSv_bytes); + head->kk_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_kk_tiles + h * L->tile_64x64_bytes); + head->qk_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_qk_tiles + h * L->tile_64x64_bytes); + head->v_inter_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_v_inter_tiles + h * L->tile_64xSv_bytes); + head->o_inter_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_o_inter_tiles + h * L->tile_64xSv_bytes); + head->inv_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_inv_row_tiles + h * L->tile_64x64_bytes); + head->a_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_a_row_tiles + h * L->tile_64x64_bytes); + head->v_prime_col_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_v_prime_col_tiles + h * L->tile_64xSv_bytes); + head->delta_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_delta_tiles + h * L->tile_64xSv_bytes); + head->delta_col_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_delta_col_tiles + h * L->tile_64xSv_bytes); + head->o_intra_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_o_intra_tiles + h * L->tile_64xSv_bytes); + head->d_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_d_row_tiles + h * L->tile_64xSv_bytes); + + head->gamma = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_gamma + h * dma_scalar_sz); + head->lambda_init_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_gamma + h * dma_scalar_sz + 128); + head->lambda_init = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_lambda_init + h * dma_scalar_sz); + head->decay_m = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_decay_m + h * decay_sz); + head->decay_a = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_decay_a + h * decay_sz); + + head->rows_kk = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_kk + h * row_vecs_sz); + head->rows_qk = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_qk + h * row_vecs_sz); + head->rows_inv = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_inv + h * row_vecs_sz); + head->rows_a = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_a + h * row_vecs_sz); + + head->vtcm_m = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_thread_scratch + h * (64 * 128)); + head->vtcm_tmp = head->vtcm_m + 32; + + head->iv1 = base_iv1 + h; + head->iv3 = iv3; + head->iq1 = fastmodulo(head->iv1, q->ne[1], &kparams->div_q1); + head->ik1 = fastmodulo(head->iv1, k->ne[1], &kparams->div_k1); + head->iq3 = fastdiv(head->iv3, &kparams->div_rq3); + head->ik3 = fastdiv(head->iv3, &kparams->div_rk3); + + head->state_in_dma = state->data + + ((uint64_t) head->iv3 * kparams->state_seq_stride + (uint64_t) head->iv1 * S_v * S_v) * sizeof(float); + + head->state_out_dma = dst_cache ? + (dst_cache->data + ((uint64_t) head->iv3 * H + head->iv1) * S_v * S_v * sizeof(float)) : + (dst->data + ((uint64_t) S_v * H * n_tokens * kparams->n_seqs + (uint64_t) (head->iv3 * H + head->iv1) * S_v * S_v) * sizeof(float)); +} + +struct htp_gdn_batch_context { + struct htp_gdn_head_ptrs * heads; + const float * vtcm_g_raw; + const float * vtcm_b_raw; + uint32_t curr_buf; + uint32_t c; + uint32_t n_batch; + uint32_t S_v; + float scale; + struct htp_ops_context * octx; + const struct htp_gdn_kernel_params * kparams; +}; + +static void gdn_hvx_init_state_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + gdn_pack_s_col_tiles(head->s_col_tiles, head->s_f16, head->s_state, bctx->S_v); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0); +} + +static inline __attribute__((unused)) HVX_Vector hvx_clamp_neg20_0(HVX_Vector v, HVX_Vector v_zero, HVX_Vector v_neg20) { + HVX_VectorPred p_gt = Q6_Q_vcmp_gt_VsfVsf(v, v_zero); + v = Q6_V_vmux_QVV(p_gt, v_zero, v); + HVX_VectorPred p_lt = Q6_Q_vcmp_gt_VsfVsf(v_neg20, v); + return Q6_V_vmux_QVV(p_lt, v_neg20, v); +} + +static inline HVX_Vector hvx_prefix_scan_f32(HVX_Vector v, HVX_Vector carry_in) { + const HVX_Vector zero = Q6_V_vzero(); + + v = hvx_vec_add_f32_f32(v, Q6_V_vlalign_VVR(v, zero, 4)); + v = hvx_vec_add_f32_f32(v, Q6_V_vlalign_VVR(v, zero, 8)); + v = hvx_vec_add_f32_f32(v, Q6_V_vlalign_VVR(v, zero, 16)); + v = hvx_vec_add_f32_f32(v, Q6_V_vlalign_VVR(v, zero, 32)); + v = hvx_vec_add_f32_f32(v, Q6_V_vlalign_VVR(v, zero, 64)); + v = hvx_vec_add_f32_f32(v, carry_in); + + return v; +} + +static inline HVX_Vector hvx_splat_last_f32(HVX_Vector v) { + return hvx_vec_repl4(Q6_V_vror_VR(v, 124)); +} + +static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t curr_buf = bctx->curr_buf; + const uint32_t S_v = bctx->S_v; + const uint32_t n_batch = bctx->n_batch; + + if (n_batch == 1) { + hvx_vmem(head->g_f32[curr_buf] + 0) = hvx_vmem(bctx->vtcm_g_raw + 0); + hvx_vmem(head->g_f32[curr_buf] + 32) = hvx_vmem(bctx->vtcm_g_raw + 32); + hvx_vmem(head->b_f32[curr_buf] + 0) = hvx_vmem(bctx->vtcm_b_raw + 0); + hvx_vmem(head->b_f32[curr_buf] + 32) = hvx_vmem(bctx->vtcm_b_raw + 32); + } else { + int32_t offsets[32] __attribute__((aligned(128))); + for (int k = 0; k < 32; ++k) { + offsets[k] = k * n_batch * sizeof(float); + } + HVX_Vector vv = *(const HVX_Vector *) offsets; + const size_t rt_g = (size_t) ((const uint8_t *) bctx->vtcm_g_raw + i * sizeof(float)); + const size_t rt_b = (size_t) ((const uint8_t *) bctx->vtcm_b_raw + i * sizeof(float)); + const size_t mu = 64 * n_batch * sizeof(float); + + Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 0), rt_g, mu, vv); + Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 32), rt_g + 32 * n_batch * sizeof(float), mu, vv); + Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 0), rt_b, mu, vv); + Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 32), rt_b + 32 * n_batch * sizeof(float), mu, vv); + } + + const uint32_t t_chunk = bctx->c * 64; + const uint32_t valid_tokens = hex_smin(64, bctx->kparams->n_tokens - t_chunk); + if (valid_tokens < 64) { + for (uint32_t t = valid_tokens; t < 64; ++t) { + head->g_f32[curr_buf][t] = 0.0f; + head->b_f32[curr_buf][t] = 0.0f; + } + const HVX_Vector vzero = Q6_V_vzero(); + for (uint32_t t = valid_tokens; t < 64; ++t) { + for (uint32_t j = 0; j < S_v; j += 32) { + hvx_vmem(head->q_f32[curr_buf] + t * S_v + j) = vzero; + hvx_vmem(head->k_f32[curr_buf] + t * S_v + j) = vzero; + hvx_vmem(head->v_f32[curr_buf] + t * S_v + j) = vzero; + } + } + } + + const HVX_Vector v_g0 = hvx_vmem(head->g_f32[curr_buf] + 0); + const HVX_Vector v_g1 = hvx_vmem(head->g_f32[curr_buf] + 32); + + HVX_Vector v_gamma0 = hvx_prefix_scan_f32(v_g0, Q6_V_vzero()); + HVX_Vector v_carry = hvx_splat_last_f32(v_gamma0); + HVX_Vector v_gamma1 = hvx_prefix_scan_f32(v_g1, v_carry); + + const HVX_Vector v_zero = Q6_V_vzero(); + const HVX_Vector v_neg20 = hvx_vec_splat_f32(-20.0f); + + hvx_vmem(head->gamma) = hvx_vec_f32_to_f16(v_gamma0, v_gamma1); + + HVX_Vector v_l0 = hvx_vec_exp_f32(hvx_clamp_neg20_0(v_gamma0, v_zero, v_neg20)); + HVX_Vector v_l1 = hvx_vec_exp_f32(hvx_clamp_neg20_0(v_gamma1, v_zero, v_neg20)); + + hvx_vmem(head->lambda_init + 0) = v_l0; + hvx_vmem(head->lambda_init + 32) = v_l1; + hvx_vmem(head->lambda_init_f16) = hvx_vec_f32_to_f16(v_l0, v_l1); + + gdn_f32_to_hmx_row_tiles_and_f16(head->k_row_tiles, head->k_prime_row_tiles, head->k_f16, + head->k_f32[curr_buf], head->lambda_init_f16, 64, S_v); + hmx_interleave_rows_to_tiles(head->k_col_tiles, head->k_f16, 64, S_v, S_v, 0, 64); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); +} + +static void gdn_hvx_phase1b_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t curr_buf = bctx->curr_buf; + const uint32_t S_v = bctx->S_v; + + gdn_f32_to_hmx_row_tiles_and_f16(head->q_row_tiles, head->q_prime_row_tiles, NULL, + head->q_f32[curr_buf], head->lambda_init_f16, 64, S_v); + + hmx_interleave_cols_to_tiles(head->k_col_tiles_64x128, head->k_f16, 64, S_v, S_v, 2, 0, 64); + + const uint16_t * gamma_u16 = (const uint16_t *) head->gamma; + const HVX_Vector v_gamma = hvx_vmem(head->gamma); + + const HVX_Vector v_zero_f16 = Q6_V_vzero(); + const HVX_Vector v_neg20_f16 = hvx_vec_splat_f16(-20.0f); + const HVX_Vector v_log2e_f16 = hvx_vec_splat_f16(1.4426950408889634f); + const HVX_Vector v_one_f16 = hvx_vec_splat_f16(1.0f); + + hvx_vmem(head->decay_m + 0) = Q6_V_vzero(); + hvx_vmem(head->decay_a + 0) = Q6_V_vand_QV(Q6_Q_vsetq2_R(2), v_one_f16); + + for (uint32_t t = 1; t < 63; t += 2) { + uint32_t t0 = t; + uint32_t t1 = t + 1; + + HVX_Vector v_gamma_t0 = Q6_Vh_vsplat_R(gamma_u16[t0]); + HVX_Vector v_gamma_t1 = Q6_Vh_vsplat_R(gamma_u16[t1]); + + HVX_Vector diff0 = hvx_vec_sub_f16_f16(v_gamma_t0, v_gamma); + HVX_Vector diff1 = hvx_vec_sub_f16_f16(v_gamma_t1, v_gamma); + + HVX_VectorPred p_gt0 = Q6_Q_vcmp_gt_VhfVhf(diff0, v_zero_f16); + HVX_VectorPred p_gt1 = Q6_Q_vcmp_gt_VhfVhf(diff1, v_zero_f16); + + diff0 = Q6_V_vmux_QVV(p_gt0, v_zero_f16, diff0); + diff1 = Q6_V_vmux_QVV(p_gt1, v_zero_f16, diff1); + + diff0 = Q6_Vhf_vmax_VhfVhf(v_neg20_f16, diff0); + diff1 = Q6_Vhf_vmax_VhfVhf(v_neg20_f16, diff1); + + HVX_Vector diff_log2e0 = hvx_vec_mul_f16_f16(diff0, v_log2e_f16); + HVX_Vector diff_log2e1 = hvx_vec_mul_f16_f16(diff1, v_log2e_f16); + + HVX_Vector v_exp0 = hvx_vec_exp2_f16(diff_log2e0); + HVX_Vector v_exp1 = hvx_vec_exp2_f16(diff_log2e1); + + HVX_VectorPred mask_lt0 = Q6_Q_vsetq2_R(2 * t0); + HVX_VectorPred mask_lt1 = Q6_Q_vsetq2_R(2 * t1); + + HVX_Vector v_m0 = Q6_V_vand_QV(mask_lt0, v_exp0); + HVX_Vector v_m1 = Q6_V_vand_QV(mask_lt1, v_exp1); + + HVX_VectorPred mask_le0 = Q6_Q_vsetq2_R(2 * (t0 + 1)); + HVX_VectorPred mask_le1 = Q6_Q_vsetq2_R(2 * (t1 + 1)); + + HVX_VectorPred mask_diag0 = Q6_Q_and_QQn(mask_le0, mask_lt0); + HVX_VectorPred mask_diag1 = Q6_Q_and_QQn(mask_le1, mask_lt1); + + HVX_Vector v_a0 = Q6_V_vmux_QVV(mask_diag0, v_one_f16, v_m0); + HVX_Vector v_a1 = Q6_V_vmux_QVV(mask_diag1, v_one_f16, v_m1); + + hvx_vmem(head->decay_m + t0 * 64) = v_m0; + hvx_vmem(head->decay_a + t0 * 64) = v_a0; + hvx_vmem(head->decay_m + t1 * 64) = v_m1; + hvx_vmem(head->decay_a + t1 * 64) = v_a1; + } + + { + HVX_Vector v_gamma_t = Q6_Vh_vsplat_R(gamma_u16[63]); + HVX_Vector diff = hvx_vec_sub_f16_f16(v_gamma_t, v_gamma); + HVX_VectorPred p_gt = Q6_Q_vcmp_gt_VhfVhf(diff, v_zero_f16); + diff = Q6_V_vmux_QVV(p_gt, v_zero_f16, diff); + diff = Q6_Vhf_vmax_VhfVhf(v_neg20_f16, diff); + + HVX_Vector diff_log2e = hvx_vec_mul_f16_f16(diff, v_log2e_f16); + HVX_Vector v_exp = hvx_vec_exp2_f16(diff_log2e); + + HVX_VectorPred mask_lt_t = Q6_Q_vsetq2_R(2 * 63); + HVX_Vector v_m = Q6_V_vand_QV(mask_lt_t, v_exp); + + HVX_VectorPred mask_le_t = Q6_Q_vcmp_eq_VhVh(v_zero_f16, v_zero_f16); + HVX_VectorPred mask_diag = Q6_Q_and_QQn(mask_le_t, mask_lt_t); + HVX_Vector v_a = Q6_V_vmux_QVV(mask_diag, v_one_f16, v_m); + + hvx_vmem(head->decay_m + 63 * 64) = v_m; + hvx_vmem(head->decay_a + 63 * 64) = v_a; + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); +} + +static void gdn_hvx_phase2_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t curr_buf = bctx->curr_buf; + + gdn_unpack_64x64_tiles_to_vectors(head->rows_kk, head->kk_tiles); + + gdn_build_inv_l_blocks( + head->inv_row_tiles, + head->rows_kk, + head->decay_m, + head->b_f32[curr_buf], + (__fp16 *) head->vtcm_m, + (__fp16 *) head->vtcm_m + HMX_FP16_TILE_N_ELMS + ); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info); +} + +static void gdn_hvx_phase3_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t curr_buf = bctx->curr_buf; + const uint32_t S_v = bctx->S_v; + + gdn_unpack_64xS_tiles_to_f32(head->v_inter_f32, head->v_inter_tiles, S_v); + + HVX_VectorAlias local_b[2]; + local_b[0].v = hvx_vmem(head->b_f32[curr_buf] + 0); + local_b[1].v = hvx_vmem(head->b_f32[curr_buf] + 32); + + for (uint32_t t = 0; t < 64; ++t) { + HVX_Vector vb = hvx_vec_splat_f32(local_b[t / 32].fp32[t % 32]); + for (uint32_t j = 0; j < S_v; j += 64) { + HVX_Vector vv0 = hvx_vmem(head->v_f32[curr_buf] + t * S_v + j + 0); + HVX_Vector vv1 = (j + 32 < S_v) ? hvx_vmem(head->v_f32[curr_buf] + t * S_v + j + 32) : Q6_V_vzero(); + HVX_Vector vi0 = hvx_vmem(head->v_inter_f32 + t * S_v + j + 0); + HVX_Vector vi1 = (j + 32 < S_v) ? hvx_vmem(head->v_inter_f32 + t * S_v + j + 32) : Q6_V_vzero(); + + HVX_Vector vp0 = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv0, vi0), vb); + HVX_Vector vp1 = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv1, vi1), vb); + + hvx_vmem(head->v_prime_f16 + t * S_v + j) = hvx_vec_f32_to_f16(vp0, vp1); + } + } + + hmx_interleave_cols_to_tiles(head->v_prime_col_tiles, head->v_prime_f16, 64, S_v, S_v, 2, 0, 64); + + gdn_unpack_64x64_tiles_to_vectors(head->rows_qk, head->qk_tiles); + for (uint32_t t = 0; t < 64; ++t) { + HVX_Vector v_decay_a = hvx_vmem(head->decay_a + t * 64); + head->rows_a[t] = hvx_vec_mul_f16_f16(head->rows_qk[t], v_decay_a); + } + gdn_pack_64x64_vectors_to_tiles(head->a_row_tiles, head->rows_a); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info); +} + +static void gdn_hvx_phase4_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t S_v = bctx->S_v; + + gdn_unpack_64xS_tiles_to_f16(head->delta_f16, head->delta_tiles, S_v); + hmx_interleave_cols_to_tiles(head->delta_col_tiles, head->delta_f16, 64, S_v, S_v, 2, 0, 64); + + const uint16_t * decay_last = (const uint16_t *) (head->decay_a + 63 * 64); + const HVX_Vector vzero = Q6_V_vzero(); + + for (uint32_t s = 0; s < 64; ++s) { + HVX_Vector vs = Q6_Vh_vsplat_R(decay_last[s]); + HVX_VectorPred p_zero = Q6_Q_vcmp_eq_VhVh(vs, vzero); + for (uint32_t j = 0; j < S_v; j += 64) { + HVX_Vector vd = hvx_vmem(head->delta_f16 + s * S_v + j); + HVX_Vector prod = hvx_vec_mul_f16_f16(vd, vs); + hvx_vmem(head->d_f16 + s * S_v + j) = Q6_V_vmux_QVV(p_zero, vzero, prod); + } + } + + gdn_pack_d_t_row_tiles(head->d_row_tiles, head->d_f16, S_v, head->vtcm_m, head->vtcm_tmp); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info); +} + +static void gdn_hvx_phase5_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t curr_buf = bctx->curr_buf; + const uint32_t S_v = bctx->S_v; + const float scale = bctx->scale; + + gdn_unpack_64xS_tiles_to_f32(head->o_inter_f32, head->o_inter_tiles, S_v); + gdn_unpack_64xS_tiles_to_f32(head->o_intra_f32, head->o_intra_tiles, S_v); + + HVX_Vector vscale = hvx_vec_splat_f32(scale); + for (uint32_t j = 0; j < 64 * S_v / 32; ++j) { + HVX_Vector vi = hvx_vmem(head->o_inter_f32 + j * 32); + HVX_Vector va = hvx_vmem(head->o_intra_f32 + j * 32); + hvx_vmem(head->o_f32[curr_buf] + j * 32) = hvx_vec_mul_f32_f32(hvx_vec_add_f32_f32(vi, va), vscale); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info); +} + +static void gdn_hvx_phase6_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_STATE, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t S_v = bctx->S_v; + const uint32_t c = bctx->c; + const uint32_t n_chunks = bctx->kparams->n_chunks; + + gdn_unpack_SxS_tiles_to_f32(head->s_update_f32, head->s_update_tiles, S_v); + + HVX_VectorAlias last_lambda; + last_lambda.v = hvx_vmem(head->lambda_init + 32); + HVX_Vector v_l_final = hvx_vec_splat_f32(last_lambda.fp32[31]); + + for (uint32_t j = 0; j < S_v * S_v / 32; ++j) { + HVX_Vector vs_old = hvx_vmem(head->s_state + j * 32); + HVX_Vector vsu = hvx_vmem(head->s_update_f32 + j * 32); + hvx_vmem(head->s_state + j * 32) = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vs_old, v_l_final), vsu); + } + + if (c + 1 < n_chunks) { + gdn_pack_s_col_tiles(head->s_col_tiles, head->s_f16, head->s_state, S_v); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_STATE, info); +} + + +static int gated_delta_net_f32_hmx_chunked( + struct htp_ops_context * octx, + const struct htp_gdn_kernel_params * kparams, + uint32_t row_start, + uint32_t nrows +) { + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * v = octx->src[2]; + const struct htp_tensor * g = octx->src[3]; + const struct htp_tensor * beta = octx->src[4]; + const struct htp_tensor * state = octx->src[5]; + const struct htp_tensor * dst = octx->dst; + const struct htp_tensor * dst_cache = octx->dsts[1]; + + const uint32_t S_v = kparams->S_v; + const uint32_t H = kparams->H; + const uint32_t n_tokens = kparams->n_tokens; + const float scale = kparams->scale; + const uint32_t chunk_size = kparams->chunk_size; + const uint32_t n_chunks = kparams->n_chunks; + const uint32_t n_sv_tiles = S_v / 32; + + struct htp_gdn_hmx_vtcm_layout L; + htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, kparams->n_heads_batch, kparams->n_threads, kparams->pipeline != 0); + + if (L.total_bytes > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + uint8_t * const vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + + float * vtcm_g_raw[2] = { + VTCM_LAYOUT_PTR(float, vtcm_base, L.off_g_raw[0]), + L.pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L.off_g_raw[1]) : VTCM_LAYOUT_PTR(float, vtcm_base, L.off_g_raw[0]) + }; + float * vtcm_b_raw[2] = { + VTCM_LAYOUT_PTR(float, vtcm_base, L.off_b_raw[0]), + L.pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L.off_b_raw[1]) : VTCM_LAYOUT_PTR(float, vtcm_base, L.off_b_raw[0]) + }; + + uint8_t * vtcm_scales_1 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L.off_scales_1); + hmx_init_column_scales(vtcm_scales_1, Q6_V_vsplat_R(0x3c00)); + + hmx_queue_t hmx_q = octx->ctx->hmx_queue; + dma_queue * dma_q = octx->ctx->dma[0]; + work_queue_t wp = octx->ctx->work_queue; + + struct htp_gdn_head_ptrs heads[8]; + struct htp_gdn_hmx_gemm_task gemm_tasks[8][9]; + + uint32_t n_batch = 1; + for (uint32_t r = row_start; r < row_start + nrows; r += n_batch) { + const uint32_t head_in_seq = fastmodulo(r, H, &kparams->div_H); + const uint32_t iv3 = fastdiv(r, &kparams->div_H); + const uint32_t heads_left_in_seq = H - head_in_seq; + const uint32_t heads_left_in_range = (row_start + nrows) - r; + n_batch = hex_smin((uint32_t) kparams->n_heads_batch, hex_smin(heads_left_in_seq, heads_left_in_range)); + + for (uint32_t h = 0; h < n_batch; ++h) { + gdn_init_head_ptrs(&heads[h], &L, vtcm_base, h, head_in_seq, iv3, + q, k, v, state, dst, dst_cache, kparams, S_v, H, n_tokens, chunk_size); + } + + struct htp_gdn_batch_context bctx; + bctx.heads = heads; + bctx.vtcm_g_raw = NULL; + bctx.vtcm_b_raw = NULL; + bctx.curr_buf = 0; + bctx.c = 0; + bctx.n_batch = n_batch; + bctx.S_v = S_v; + bctx.scale = scale; + bctx.octx = octx; + bctx.kparams = kparams; + + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_push(dma_q, dma_make_data(heads[h].s_state, heads[h].state_in_dma), + S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), S_v); + } + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_pop(dma_q); + } + + if (n_chunks > 0) { + work_queue_run(wp, gdn_hvx_init_state_worker, &bctx, n_batch); + + const uint32_t chunk0_tokens = hex_smin(chunk_size, n_tokens); + for (uint32_t h = 0; h < n_batch; ++h) { + gdn_dma_push_chunk_inputs(dma_q, heads[h].q_f32[0], heads[h].k_f32[0], heads[h].v_f32[0], + q, k, v, heads[h].iq3, heads[h].iq1, heads[h].ik3, heads[h].ik1, + heads[h].iv3, heads[h].iv1, 0, chunk0_tokens, S_v); + } + gdn_dma_push_chunk_gb(dma_q, vtcm_g_raw[0], vtcm_b_raw[0], g, beta, iv3, head_in_seq, 0, chunk0_tokens, n_batch); + } + + for (uint32_t c = 0; c < n_chunks; ++c) { + const uint32_t curr_buf = c & 1; + const uint32_t next_buf = (c + 1) & 1; + const uint32_t t_chunk = c * chunk_size; + + bctx.curr_buf = curr_buf; + bctx.c = c; + bctx.vtcm_g_raw = vtcm_g_raw[curr_buf]; + bctx.vtcm_b_raw = vtcm_b_raw[curr_buf]; + + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + } + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + if (c + 1 < n_chunks) { + const uint32_t next_t_chunk = (c + 1) * chunk_size; + const uint32_t next_tokens = hex_smin(chunk_size, n_tokens - next_t_chunk); + for (uint32_t h = 0; h < n_batch; ++h) { + gdn_dma_push_chunk_inputs(dma_q, heads[h].q_f32[next_buf], heads[h].k_f32[next_buf], heads[h].v_f32[next_buf], + q, k, v, heads[h].iq3, heads[h].iq1, heads[h].ik3, heads[h].ik1, + heads[h].iv3, heads[h].iv1, next_t_chunk, next_tokens, S_v); + } + gdn_dma_push_chunk_gb(dma_q, vtcm_g_raw[next_buf], vtcm_b_raw[next_buf], + g, beta, iv3, head_in_seq, next_t_chunk, next_tokens, n_batch); + } + + if (c > 0) { + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_pop(dma_q); + } + } + + work_queue_run(wp, gdn_hvx_phase1a_worker, &bctx, n_batch); + + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][0], heads[h].k_row_tiles, heads[h].k_col_tiles, heads[h].kk_tiles, 2, 2, n_sv_tiles, vtcm_scales_1); + } + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][2], heads[h].k_prime_row_tiles, heads[h].s_col_tiles, heads[h].v_inter_tiles, 2, n_sv_tiles, n_sv_tiles, vtcm_scales_1); + } + + work_queue_run(wp, gdn_hvx_phase1b_worker, &bctx, n_batch); + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][1], heads[h].q_row_tiles, heads[h].k_col_tiles, heads[h].qk_tiles, 2, 2, n_sv_tiles, vtcm_scales_1); + } + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][3], heads[h].q_prime_row_tiles, heads[h].s_col_tiles, heads[h].o_inter_tiles, 2, n_sv_tiles, n_sv_tiles, vtcm_scales_1); + } + + work_queue_run(wp, gdn_hvx_phase2_worker, &bctx, n_batch); + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task( + hmx_q, &gemm_tasks[h][7], + (__fp16 *) heads[h].vtcm_m, + heads[h].inv_row_tiles + 0 * HMX_FP16_TILE_N_ELMS, + (__fp16 *) heads[h].vtcm_tmp, + 1, 1, 1, vtcm_scales_1 + ); + } + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task( + hmx_q, &gemm_tasks[h][8], + (__fp16 *) heads[h].vtcm_m + HMX_FP16_TILE_N_ELMS, + (__fp16 *) heads[h].vtcm_tmp, + heads[h].inv_row_tiles + 2 * HMX_FP16_TILE_N_ELMS, + 1, 1, 1, vtcm_scales_1 + ); + } + + work_queue_run(wp, gdn_hvx_phase3_worker, &bctx, n_batch); + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][4], heads[h].inv_row_tiles, heads[h].v_prime_col_tiles, heads[h].delta_tiles, 2, n_sv_tiles, 2, vtcm_scales_1); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + work_queue_run(wp, gdn_hvx_phase4_worker, &bctx, n_batch); + + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][5], heads[h].a_row_tiles, heads[h].delta_col_tiles, heads[h].o_intra_tiles, 2, n_sv_tiles, 2, vtcm_scales_1); + } + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][6], heads[h].d_row_tiles, heads[h].k_col_tiles_64x128, heads[h].s_update_tiles, n_sv_tiles, n_sv_tiles, 2, vtcm_scales_1); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + work_queue_run(wp, gdn_hvx_phase5_worker, &bctx, n_batch); + + const uint32_t valid_tokens = hex_smin(chunk_size, n_tokens - t_chunk); + for (uint32_t h = 0; h < n_batch; ++h) { + const dma_addr_t attn_chunk_dma = dst->data + + ((uint64_t) heads[h].iv3 * n_tokens * H + (uint64_t) t_chunk * H + heads[h].iv1) * S_v * sizeof(float); + dma_queue_push(dma_q, dma_make_data(attn_chunk_dma, heads[h].o_f32[curr_buf]), + dst->nb[1], S_v * sizeof(float), S_v * sizeof(float), valid_tokens); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + work_queue_run(wp, gdn_hvx_phase6_worker, &bctx, n_batch); + } + + if (n_chunks > 0) { + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_pop(dma_q); + } + } + + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_push(dma_q, dma_make_data(heads[h].state_out_dma, heads[h].s_state), + S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), S_v); + } + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_pop(dma_q); + } + } + + dma_queue_flush(dma_q); + return HTP_STATUS_OK; +} int op_gated_delta_net(struct htp_ops_context * octx) { const struct htp_tensor * q = octx->src[0]; @@ -1085,10 +2278,6 @@ int op_gated_delta_net(struct htp_ops_context * octx) { const struct htp_tensor * state = octx->src[5]; const struct htp_tensor * dst = octx->dst; - if (!q || !k || !v || !g || !beta || !state || !dst) { - return HTP_STATUS_INVAL_PARAMS; - } - if (q->type != HTP_TYPE_F32 || k->type != HTP_TYPE_F32 || v->type != HTP_TYPE_F32 || g->type != HTP_TYPE_F32 || beta->type != HTP_TYPE_F32 || state->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) { @@ -1120,23 +2309,127 @@ int op_gated_delta_net(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; + for (int i = 0; i < 5; i++) { + if (htp_tensor_is_extended(octx->src[i])) { + return HTP_STATUS_NO_SUPPORT; + } + } + if (htp_tensor_is_extended(octx->dst)) { + return HTP_STATUS_NO_SUPPORT; + } + if (octx->dsts[1]) { + const struct htp_tensor * dst_cache = octx->dsts[1]; + if (dst_cache->type != HTP_TYPE_F32 || htp_tensor_is_extended(dst_cache)) { + return HTP_STATUS_NO_SUPPORT; + } } - struct htp_gdn_context gctx; - gctx.octx = octx; - gctx.rows_per_thread = (H * n_seqs + octx->n_threads - 1) / octx->n_threads; - gctx.state_bytes = (size_t) S_v * S_v * sizeof(float); + const struct htp_gdn_kernel_params * kparams = (const struct htp_gdn_kernel_params *) octx->kernel_params; + struct htp_gdn_kernel_params kparams_local; + if (!kparams || kparams->S_v == 0) { + const uint32_t rq3 = n_seqs / q->ne[3]; + const uint32_t rk3 = n_seqs / k->ne[3]; + const uint32_t total_rows = H * n_seqs; + uint32_t n_threads = (total_rows < octx->n_threads) ? total_rows : octx->n_threads; + if (n_threads == 0) { + n_threads = 1; + } + + memset(&kparams_local, 0, sizeof(kparams_local)); + kparams_local.n_threads = n_threads; + kparams_local.S_v = S_v; + kparams_local.H = H; + kparams_local.n_tokens = n_tokens; + kparams_local.n_seqs = n_seqs; + kparams_local.K = K; + kparams_local.total_rows = total_rows; + kparams_local.rows_per_thread = (total_rows + n_threads - 1) / n_threads; + const bool can_use_hmx = (octx->ctx->hmx_enabled) && + (S_v % 64 == 0) && + (n_tokens >= HTP_GDN_MIN_TOKENS) && + (g->ne[0] == 1) && + (K == 1); + + struct htp_gdn_hmx_vtcm_layout hmx_layout_local; + struct htp_gdn_vtcm_layout hvx_layout_local; + uint32_t n_heads_batch = 1; + + if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout_local, S_v, HTP_GDN_CHUNK_SIZE, total_rows, octx->ctx->vtcm_size, n_threads, true, &n_heads_batch)) { + kparams_local.kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED; + kparams_local.pipeline = hmx_layout_local.pipeline ? 1 : 0; + kparams_local.chunk_size = HTP_GDN_CHUNK_SIZE; + kparams_local.n_chunks = (n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE; + kparams_local.n_heads_batch = (uint16_t) n_heads_batch; + kparams_local.vtcm_size = (uint32_t) hmx_layout_local.total_bytes; + kparams_local.state_aligned = (uint32_t) hmx_layout_local.state_f32_bytes; + kparams_local.vtcm_per_thread = (uint32_t) (hmx_layout_local.total_bytes / (n_threads > 0 ? n_threads : 1)); + } else { + htp_gdn_vtcm_layout_build(&hvx_layout_local, S_v, n_threads); + kparams_local.kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT; + kparams_local.pipeline = 0; + kparams_local.n_heads_batch = 1; + kparams_local.state_aligned = (uint32_t) hvx_layout_local.state_aligned; + kparams_local.vtcm_per_thread = (uint32_t) hvx_layout_local.bytes_per_thread; + kparams_local.vtcm_size = (uint32_t) hvx_layout_local.total_bytes; + } + kparams_local.kda = (g->ne[0] == S_v) ? 1 : 0; + kparams_local.scale = 1.0f / sqrtf((float) S_v); + kparams_local.state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float)); + kparams_local.state_size_per_snap = S_v * S_v * H * n_seqs; + + kparams_local.div_H = init_fastdiv_values(H); + kparams_local.div_q1 = init_fastdiv_values(q->ne[1]); + kparams_local.div_k1 = init_fastdiv_values(k->ne[1]); + kparams_local.div_rq3 = init_fastdiv_values(rq3); + kparams_local.div_rk3 = init_fastdiv_values(rk3); + kparams_local.div_n_threads = init_fastdiv_values(n_threads); + + kparams = &kparams_local; + } + + const uint32_t total_rows = kparams->total_rows; + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && + ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, + can_split ? 1 : 0, + octx->ctx->mdev.idx, + octx->ctx->mdev.count, + &octx->ctx->mdev.count_div + ); + row_start = range.start; + nrows = range.count; + } else if (octx->op_params[1] != 0) { + row_start = octx->op_params[1]; + nrows = octx->op_params[2]; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } - size_t state_aligned = (size_t) S_v * S_v * sizeof(float); - state_aligned = (state_aligned + 127) & ~(size_t)127; + if (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) { + return gated_delta_net_f32_hmx_chunked(octx, kparams, row_start, nrows); + } - assert(octx->ctx->vtcm_base != NULL); - assert(octx->ctx->vtcm_size >= 2 * state_aligned * octx->n_threads); + const uint32_t n_threads = (nrows < kparams->n_threads) ? nrows : kparams->n_threads; + struct htp_gdn_context gctx; + gctx.octx = octx; + gctx.kparams = kparams; + gctx.row_start = row_start; + gctx.nrows = nrows; gctx.vtcm_base = octx->ctx->vtcm_base; - gctx.vtcm_per_thread = 2 * state_aligned; + + htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_threads); + + if (gctx.layout.total_bytes > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } FARF(HIGH, "gated-delta-net-f32: q(%ux%ux%ux%u) k(%ux%ux%ux%u) v(%ux%ux%ux%u) state(%ux%ux%ux%u) -> (%ux%ux%ux%u) : " "vtcm-size %zu n_threads %u\n", @@ -1145,12 +2438,12 @@ int op_gated_delta_net(struct htp_ops_context * octx) { v->ne[0], v->ne[1], v->ne[2], v->ne[3], state->ne[0], state->ne[1], state->ne[2], state->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - gctx.vtcm_per_thread * octx->n_threads, octx->n_threads); + gctx.layout.total_bytes, n_threads); if (n_tokens == 1) { - worker_pool_run_func(octx->ctx->worker_pool, gated_delta_net_f32_tg_thread, &gctx, octx->n_threads); + work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_tg_thread, &gctx, n_threads); } else { - worker_pool_run_func(octx->ctx->worker_pool, gated_delta_net_f32_pp_thread, &gctx, octx->n_threads); + work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_pp_thread, &gctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h new file mode 100644 index 000000000000..32fb7d24bfdd --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h @@ -0,0 +1,301 @@ +#ifndef HTP_GATED_DELTA_NET_OPS_H +#define HTP_GATED_DELTA_NET_OPS_H + +#include <stdint.h> +#include <stddef.h> +#include <stdbool.h> + +#include "hex-fastdiv.h" +#include "hex-common.h" +#include "htp-vtcm.h" + +#define HTP_GDN_MAX_SV 128 +#define HTP_GDN_CHUNK_SIZE 64 +#define HTP_GDN_MIN_TOKENS 8 + +#ifndef HMX_FP16_TILE_SIZE +#define HMX_FP16_TILE_SIZE 2048 +#endif + +enum htp_gdn_kernel_type { + HTP_GDN_KERNEL_HVX_RECURRENT = 0, + HTP_GDN_KERNEL_HMX_CHUNKED = 1, +}; + +struct htp_gdn_kernel_params { + uint8_t kernel_type; + uint8_t pipeline; + uint16_t chunk_size; + uint16_t n_chunks; + uint16_t n_heads_batch; + + uint32_t n_threads; + uint32_t S_v; + uint32_t H; + uint32_t n_tokens; + uint32_t n_seqs; + uint32_t K; + + uint32_t total_rows; + uint32_t row_start; + uint32_t nrows; + uint32_t rows_per_thread; + + uint32_t kda; + uint32_t state_aligned; + uint32_t vtcm_per_thread; + uint32_t vtcm_size; + uint32_t state_seq_stride; + uint32_t state_size_per_snap; + + float scale; + + struct fastdiv_values div_H; + struct fastdiv_values div_q1; + struct fastdiv_values div_k1; + struct fastdiv_values div_rq3; + struct fastdiv_values div_rk3; + struct fastdiv_values div_n_threads; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_gdn_vtcm_layout { + size_t state_aligned; + size_t bytes_per_thread; + size_t total_bytes; +}; + +static inline void htp_gdn_vtcm_layout_build( + struct htp_gdn_vtcm_layout * layout, + uint32_t S_v, + uint32_t n_threads +) { + size_t state_bytes = (size_t) S_v * S_v * sizeof(float); + layout->state_aligned = hex_round_up(state_bytes, 128); + layout->bytes_per_thread = 2 * layout->state_aligned; + layout->total_bytes = layout->bytes_per_thread * n_threads; +} + +struct htp_gdn_hmx_vtcm_layout { + size_t off_s_state; + size_t off_s_f16; + size_t off_s_col_tiles; + size_t off_s_update_f32; + size_t off_s_update_tiles; + + size_t off_q_f32[2]; + size_t off_k_f32[2]; + size_t off_v_f32[2]; + size_t off_g_f32[2]; + size_t off_b_f32[2]; + size_t off_g_raw[2]; + size_t off_b_raw[2]; + size_t off_o_f32[2]; + + size_t off_v_inter_f32; + size_t off_o_inter_f32; + size_t off_o_intra_f32; + size_t off_k_f16; + size_t off_v_prime_f16; + size_t off_delta_f16; + size_t off_d_f16; + + size_t off_q_row_tiles; + size_t off_q_prime_row_tiles; + size_t off_k_row_tiles; + size_t off_k_col_tiles; + size_t off_k_prime_row_tiles; + size_t off_k_col_tiles_64x128; + size_t off_kk_tiles; + size_t off_qk_tiles; + size_t off_v_inter_tiles; + size_t off_o_inter_tiles; + size_t off_inv_row_tiles; + size_t off_a_row_tiles; + size_t off_v_prime_col_tiles; + size_t off_delta_tiles; + size_t off_delta_col_tiles; + size_t off_o_intra_tiles; + size_t off_d_row_tiles; + + size_t off_gamma; + size_t off_lambda_init; + size_t off_decay_m; + size_t off_decay_a; + size_t off_rows_kk; + size_t off_rows_qk; + size_t off_rows_inv; + size_t off_rows_a; + + size_t off_thread_scratch; + size_t off_scales_1; + + size_t state_f32_bytes; + size_t state_f16_bytes; + size_t state_tiles_bytes; + size_t dma_chunk_bytes; + size_t act_f16_bytes; + size_t tile_64xSv_bytes; + size_t tile_64x64_bytes; + + uint32_t n_heads_batch; + uint32_t n_threads; + bool pipeline; + size_t total_bytes; +}; + +static inline void htp_gdn_hmx_vtcm_layout_build( + struct htp_gdn_hmx_vtcm_layout * L, + uint32_t S_v, + uint32_t chunk_size, + uint32_t n_heads_batch, + uint32_t n_threads, + bool pipeline +) { + memset(L, 0, sizeof(*L)); + L->n_heads_batch = n_heads_batch; + L->n_threads = n_threads; + L->pipeline = pipeline; + + const size_t bh = (size_t) n_heads_batch; + const size_t nth = (size_t) (n_threads > 0 ? n_threads : 1); + + const size_t state_f32_sz = hex_round_up(S_v * S_v * sizeof(float), 2048); + const size_t state_f16_sz = hex_round_up(S_v * S_v * sizeof(__fp16), 2048); + const size_t n_sv_tiles = S_v / 32; + const size_t state_tiles_sz = n_sv_tiles * n_sv_tiles * 2048; + + const size_t dma_chunk_sz = hex_round_up(chunk_size * S_v * sizeof(float), 2048); + const size_t dma_scalar_sz = hex_round_up(chunk_size * sizeof(float), 128); + + const size_t act_f16_sz = hex_round_up(chunk_size * S_v * sizeof(__fp16), 2048); + const size_t tile_64xSv_sz = 2 * n_sv_tiles * 2048; + const size_t tile_64x64_sz = 4 * 2048; + + const size_t decay_sz = 64 * 64 * sizeof(__fp16); + const size_t row_vecs_sz = 64 * 128; + + L->state_f32_bytes = state_f32_sz; + L->state_f16_bytes = state_f16_sz; + L->state_tiles_bytes = state_tiles_sz; + L->dma_chunk_bytes = dma_chunk_sz; + L->act_f16_bytes = act_f16_sz; + L->tile_64xSv_bytes = tile_64xSv_sz; + L->tile_64x64_bytes = tile_64x64_sz; + + size_t off = 0; + + VTCM_LAYOUT_ALLOC(off, off_s_state, bh * state_f32_sz); + VTCM_LAYOUT_ALLOC(off, off_s_f16, bh * state_f16_sz); + off = hex_align_up(off, HMX_FP16_TILE_SIZE); + VTCM_LAYOUT_ALLOC(off, off_s_col_tiles, bh * state_tiles_sz); + VTCM_LAYOUT_ALLOC(off, off_s_update_f32, bh * state_f32_sz); + off = hex_align_up(off, HMX_FP16_TILE_SIZE); + VTCM_LAYOUT_ALLOC(off, off_s_update_tiles, bh * state_tiles_sz); + + VTCM_LAYOUT_ALLOC(off, off_q_f32[0], bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_q_f32[1], bh * dma_chunk_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_k_f32[0], bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_k_f32[1], bh * dma_chunk_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_v_f32[0], bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_v_f32[1], bh * dma_chunk_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_g_f32[0], bh * dma_scalar_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_f32[1], bh * dma_scalar_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_b_f32[0], bh * dma_scalar_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_b_f32[1], bh * dma_scalar_sz, pipeline); + const size_t raw_gb_sz = hex_round_up(bh * chunk_size * sizeof(float), 128); + VTCM_LAYOUT_ALLOC(off, off_g_raw[0], raw_gb_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_raw[1], raw_gb_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_b_raw[0], raw_gb_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_b_raw[1], raw_gb_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_o_f32[0], bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_o_f32[1], bh * dma_chunk_sz, pipeline); + + VTCM_LAYOUT_ALLOC(off, off_v_inter_f32, bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC(off, off_o_inter_f32, bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC(off, off_o_intra_f32, bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC(off, off_k_f16, bh * act_f16_sz); + VTCM_LAYOUT_ALLOC(off, off_v_prime_f16, bh * act_f16_sz); + VTCM_LAYOUT_ALLOC(off, off_delta_f16, bh * act_f16_sz); + VTCM_LAYOUT_ALLOC(off, off_d_f16, bh * act_f16_sz); + + off = hex_align_up(off, HMX_FP16_TILE_SIZE); + VTCM_LAYOUT_ALLOC(off, off_q_row_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_q_prime_row_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_k_row_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_k_col_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_k_prime_row_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_k_col_tiles_64x128, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_kk_tiles, bh * tile_64x64_sz); + VTCM_LAYOUT_ALLOC(off, off_qk_tiles, bh * tile_64x64_sz); + VTCM_LAYOUT_ALLOC(off, off_v_inter_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_o_inter_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_inv_row_tiles, bh * tile_64x64_sz); + VTCM_LAYOUT_ALLOC(off, off_a_row_tiles, bh * tile_64x64_sz); + VTCM_LAYOUT_ALLOC(off, off_v_prime_col_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_delta_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_delta_col_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_o_intra_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_d_row_tiles, bh * tile_64xSv_sz); + + VTCM_LAYOUT_ALLOC(off, off_gamma, bh * hex_round_up(chunk_size * sizeof(float), 128)); + VTCM_LAYOUT_ALLOC(off, off_lambda_init, bh * hex_round_up(chunk_size * sizeof(float), 128)); + VTCM_LAYOUT_ALLOC(off, off_decay_m, bh * decay_sz); + VTCM_LAYOUT_ALLOC(off, off_decay_a, bh * decay_sz); + VTCM_LAYOUT_ALLOC(off, off_rows_kk, bh * row_vecs_sz); + VTCM_LAYOUT_ALLOC(off, off_rows_qk, bh * row_vecs_sz); + VTCM_LAYOUT_ALLOC(off, off_rows_inv, bh * row_vecs_sz); + VTCM_LAYOUT_ALLOC(off, off_rows_a, bh * row_vecs_sz); + + const size_t thread_scratch_sz = 64 * 128; + off = hex_align_up(off, HMX_FP16_TILE_SIZE); + VTCM_LAYOUT_ALLOC(off, off_thread_scratch, nth * thread_scratch_sz); + off = hex_align_up(off, HMX_FP16_TILE_SIZE); + VTCM_LAYOUT_ALLOC(off, off_scales_1, HMX_FP16_TILE_SIZE); + + L->total_bytes = off; +} + +static inline bool htp_gdn_hmx_solve_layout( + struct htp_gdn_hmx_vtcm_layout * layout_out, + uint32_t S_v, + uint32_t chunk_size, + uint32_t total_rows, + size_t vtcm_budget, + uint32_t n_threads, + bool pipeline, + uint32_t * n_heads_batch_out +) { + uint32_t max_batch = 8; + if (max_batch > total_rows) { + max_batch = total_rows; + } + if (max_batch > n_threads) { + max_batch = n_threads; + } + static const uint32_t candidates[] = { 8, 6, 4, 2, 1 }; + for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) { + uint32_t bh = candidates[i]; + if (bh > max_batch) { + continue; + } + struct htp_gdn_hmx_vtcm_layout L; + htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, bh, n_threads, pipeline); + if (L.total_bytes <= vtcm_budget) { + *layout_out = L; + *n_heads_batch_out = bh; + return true; + } + } + if (pipeline) { + return htp_gdn_hmx_solve_layout(layout_out, S_v, chunk_size, total_rows, vtcm_budget, n_threads, false, n_heads_batch_out); + } + return false; +} + +#endif // HTP_GATED_DELTA_NET_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c index a87962d22910..f51e00c15f36 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c @@ -10,6 +10,8 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "dma-queue.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -23,9 +25,12 @@ struct get_rows_context { const struct htp_get_rows_kernel_params * kparams; struct htp_get_rows_vtcm_layout vtcm_layout; uint8_t * vtcm_base; + uint32_t task_start; + uint32_t tasks; + uint32_t tasks_per_thread; }; -#define get_rows_preamble \ +#define get_rows_preamble \ const uint32_t ne00 = octx->src[0]->ne[0]; \ const uint32_t ne01 = octx->src[0]->ne[1]; \ const uint32_t ne02 = octx->src[0]->ne[2]; \ @@ -55,140 +60,139 @@ struct get_rows_context { \ const uint32_t nr = ne10 * ne11 * ne12; -#define GET_ROWS_THREAD_ST_FN(IDX_TYPE) \ -static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ - struct get_rows_context * grctx = (struct get_rows_context *)data; \ - struct htp_ops_context * octx = grctx->octx; \ - const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ - get_rows_preamble; \ - const uint32_t dr = kparams->tasks_per_thread; \ - const uint32_t ir0 = dr * ith; \ - if (ir0 >= kparams->total_tasks) { \ - return; \ - } \ - const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ - const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00); \ - dma_queue * dma_queue = octx->ctx->dma[ith]; \ - for (uint32_t i = ir0; i < ir1; ++i) { \ - const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); \ - const uint32_t rem = i - i12 * ne11 * ne10; \ - const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ - const uint32_t i10 = rem - i11 * ne10; \ - const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \ - const uint32_t i01 = (uint32_t)*src1_ptr; \ - assert(i01 < ne01); \ - const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \ - const uint32_t i02 = i11 - q02 * ne02; \ - const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \ - const uint32_t i03 = i12 - q03 * ne03; \ - const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; \ - const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; \ - while (!dma_queue_push(dma_queue, dma_make_ptr((void *)dst_ptr, (const void *)src0_ptr), nb1, nb01, \ - row_size_bytes, 1)) { \ - dma_queue_pop(dma_queue); \ - } \ - } \ - dma_queue_flush(dma_queue); \ +#define GET_ROWS_THREAD_ST_FN(IDX_TYPE) \ +static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ + struct get_rows_context * grctx = (struct get_rows_context *)data; \ + struct htp_ops_context * octx = grctx->octx; \ + const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ + get_rows_preamble; \ + const uint32_t dr = grctx->tasks_per_thread; \ + const uint32_t ir0 = grctx->task_start + dr * ith; \ + if (ir0 >= grctx->task_start + grctx->tasks) { \ + return; \ + } \ + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \ + const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00); \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + for (uint32_t i = ir0; i < ir1; ++i) { \ + const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); \ + const uint32_t rem = i - i12 * ne11 * ne10; \ + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ + const uint32_t i10 = rem - i11 * ne10; \ + const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(uintptr_t)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \ + const uint32_t i01 = (uint32_t)*src1_ptr; \ + assert(i01 < ne01); \ + const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \ + const uint32_t i02 = i11 - q02 * ne02; \ + const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \ + const uint32_t i03 = i12 - q03 * ne03; \ + const dma_addr_t src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; \ + const dma_addr_t dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; \ + while (!dma_queue_push(dma_q, dma_make_data(dst_data, src0_data), nb1, nb01, \ + row_size_bytes, 1)) { \ + dma_queue_pop(dma_q); \ + } \ + } \ + dma_queue_flush(dma_q); \ } GET_ROWS_THREAD_ST_FN(int32_t) GET_ROWS_THREAD_ST_FN(int64_t) -#define GET_ROWS_THREAD_DT_FN(TYPE_NAME, SRC0_SIZE_EXPR, IDX_TYPE, COMPUTE_EXPR) \ -static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ - struct get_rows_context * grctx = (struct get_rows_context *)data; \ - struct htp_ops_context * octx = grctx->octx; \ - const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ - get_rows_preamble; \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - const uint32_t dr = kparams->tasks_per_thread; \ - const uint32_t ir0 = dr * ith; \ - if (ir0 >= kparams->total_tasks) { \ - return; \ - } \ - const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ - const uint32_t chunks_per_row = kparams->chunks_per_row; \ - const uint32_t chunk_size = kparams->chunk_size; \ - dma_queue * dma_queue = octx->ctx->dma[ith]; \ - const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; \ - uint8_t * vtcm_src0 = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \ - uint8_t * vtcm_dst = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \ - for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) { \ - const uint32_t i = ir0 + step; \ - const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \ - const uint32_t chunk_idx = i - row_idx * chunks_per_row; \ - const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \ - const uint32_t rem = row_idx - i12 * ne11 * ne10; \ - const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ - const uint32_t i10 = rem - i11 * ne10; \ - const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \ - const uint32_t i01 = (uint32_t)*src1_ptr; \ - assert(i01 < ne01); \ - const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \ - const uint32_t i02 = i11 - q02 * ne02; \ - const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \ - const uint32_t i03 = i12 - q03 * ne03; \ - const uint32_t offset = chunk_idx * chunk_size; \ - const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \ - const uint32_t cur_src0_bytes = SRC0_SIZE_EXPR(cur_elems); \ - const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \ - const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03 + SRC0_SIZE_EXPR(offset); \ - dma_queue_push(dma_queue, \ - dma_make_ptr((void *)(uintptr_t)octx->dst->data, \ - vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \ - cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 0); \ - dma_queue_push(dma_queue, \ - dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \ - (const void *)src0_ptr), \ - vtcm_layout->src0_spad_half_size, cur_src0_bytes, cur_src0_bytes, 1); \ - } \ - for (uint32_t step = 0; step < ir1 - ir0; ++step) { \ - const uint32_t i = ir0 + step; \ - void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \ - void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \ - const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \ - const uint32_t chunk_idx = i - row_idx * chunks_per_row; \ - const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \ - const uint32_t rem = row_idx - i12 * ne11 * ne10; \ - const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ - const uint32_t i10 = rem - i11 * ne10; \ - const uint32_t offset = chunk_idx * chunk_size; \ - const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \ - const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, i); \ - COMPUTE_EXPR; \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, i); \ - const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float); \ - dma_queue_push(dma_queue, \ - dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \ - cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 1); \ - const uint32_t next_step = step + 2; \ - if (next_step < ir1 - ir0) { \ - const uint32_t pi = ir0 + next_step; \ - const uint32_t prow_idx = fastdiv(pi, &kparams->div_chunks_per_row); \ - const uint32_t pchunk_idx = pi - prow_idx * chunks_per_row; \ - const uint32_t pi12 = fastdiv(prow_idx, &kparams->div_ne10_ne11); \ - const uint32_t prem = prow_idx - pi12 * ne11 * ne10; \ - const uint32_t pi11 = fastdiv(prem, &kparams->div_ne10); \ - const uint32_t pi10 = prem - pi11 * ne10; \ - const IDX_TYPE * psrc1_ptr = (const IDX_TYPE *)(octx->src[1]->data + pi10*nb10 + pi11*nb11 + pi12*nb12); \ - const uint32_t pi01 = (uint32_t)*psrc1_ptr; \ - assert(pi01 < ne01); \ - const uint32_t pq02 = fastdiv(pi11, &kparams->div_ne02); \ - const uint32_t pi02 = pi11 - pq02 * ne02; \ - const uint32_t pq03 = fastdiv(pi12, &kparams->div_ne03); \ - const uint32_t pi03 = pi12 - pq03 * ne03; \ - const uint32_t poffset = pchunk_idx * chunk_size; \ - const uint32_t pcur_elems = (poffset < ne00) ? MIN(chunk_size, ne00 - poffset) : 0; \ - const uint32_t pcur_src0_bytes = SRC0_SIZE_EXPR(pcur_elems); \ - const uintptr_t psrc0_ptr = \ - octx->src[0]->data + pi01*nb01 + pi02*nb02 + pi03*nb03 + SRC0_SIZE_EXPR(poffset); \ - dma_queue_push(dma_queue, \ - dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \ - vtcm_layout->src0_spad_half_size, pcur_src0_bytes, pcur_src0_bytes, 1); \ - } \ - } \ - dma_queue_flush(dma_queue); \ +#define GET_ROWS_THREAD_DT_FN(TYPE_NAME, SRC0_SIZE_EXPR, IDX_TYPE, COMPUTE_EXPR) \ +static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ + struct get_rows_context * grctx = (struct get_rows_context *)data; \ + struct htp_ops_context * octx = grctx->octx; \ + const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ + get_rows_preamble; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + const uint32_t dr = grctx->tasks_per_thread; \ + const uint32_t ir0 = grctx->task_start + dr * ith; \ + if (ir0 >= grctx->task_start + grctx->tasks) { \ + return; \ + } \ + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \ + const uint32_t chunks_per_row = kparams->chunks_per_row; \ + const uint32_t chunk_size = kparams->chunk_size; \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; \ + uint8_t * vtcm_src0 = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \ + uint8_t * vtcm_dst = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \ + for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) { \ + const uint32_t i = ir0 + step; \ + const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \ + const uint32_t chunk_idx = i - row_idx * chunks_per_row; \ + const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \ + const uint32_t rem = row_idx - i12 * ne11 * ne10; \ + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ + const uint32_t i10 = rem - i11 * ne10; \ + const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(uintptr_t)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \ + const uint32_t i01 = (uint32_t)*src1_ptr; \ + assert(i01 < ne01); \ + const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \ + const uint32_t i02 = i11 - q02 * ne02; \ + const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \ + const uint32_t i03 = i12 - q03 * ne03; \ + const uint32_t offset = chunk_idx * chunk_size; \ + const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \ + const uint32_t cur_src0_bytes = SRC0_SIZE_EXPR(cur_elems); \ + const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \ + const dma_addr_t src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03 + SRC0_SIZE_EXPR(offset); \ + dma_queue_push(dma_q, \ + dma_make_data(octx->dst->data, \ + vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \ + cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 0); \ + dma_queue_push(dma_q, \ + dma_make_data(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size, src0_data), \ + vtcm_layout->src0_spad_half_size, cur_src0_bytes, cur_src0_bytes, 1); \ + } \ + for (uint32_t step = 0; step < ir1 - ir0; ++step) { \ + const uint32_t i = ir0 + step; \ + void * dst_spad = (void *) dma_queue_pop(dma_q).src; \ + void * src_spad = (void *) dma_queue_pop(dma_q).dst; \ + const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \ + const uint32_t chunk_idx = i - row_idx * chunks_per_row; \ + const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \ + const uint32_t rem = row_idx - i12 * ne11 * ne10; \ + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ + const uint32_t i10 = rem - i11 * ne10; \ + const uint32_t offset = chunk_idx * chunk_size; \ + const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \ + const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, i); \ + COMPUTE_EXPR; \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, i); \ + const dma_addr_t dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float); \ + dma_queue_push(dma_q, \ + dma_make_data(dst_data, dst_spad), \ + cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 1); \ + const uint32_t next_step = step + 2; \ + if (next_step < ir1 - ir0) { \ + const uint32_t pi = ir0 + next_step; \ + const uint32_t prow_idx = fastdiv(pi, &kparams->div_chunks_per_row); \ + const uint32_t pchunk_idx = pi - prow_idx * chunks_per_row; \ + const uint32_t pi12 = fastdiv(prow_idx, &kparams->div_ne10_ne11); \ + const uint32_t prem = prow_idx - pi12 * ne11 * ne10; \ + const uint32_t pi11 = fastdiv(prem, &kparams->div_ne10); \ + const uint32_t pi10 = prem - pi11 * ne10; \ + const IDX_TYPE * psrc1_ptr = (const IDX_TYPE *)(uintptr_t)(octx->src[1]->data + pi10*nb10 + pi11*nb11 + pi12*nb12); \ + const uint32_t pi01 = (uint32_t)*psrc1_ptr; \ + assert(pi01 < ne01); \ + const uint32_t pq02 = fastdiv(pi11, &kparams->div_ne02); \ + const uint32_t pi02 = pi11 - pq02 * ne02; \ + const uint32_t pq03 = fastdiv(pi12, &kparams->div_ne03); \ + const uint32_t pi03 = pi12 - pq03 * ne03; \ + const uint32_t poffset = pchunk_idx * chunk_size; \ + const uint32_t pcur_elems = (poffset < ne00) ? MIN(chunk_size, ne00 - poffset) : 0; \ + const uint32_t pcur_src0_bytes = SRC0_SIZE_EXPR(pcur_elems); \ + const dma_addr_t psrc0_data = \ + octx->src[0]->data + pi01*nb01 + pi02*nb02 + pi03*nb03 + SRC0_SIZE_EXPR(poffset); \ + dma_queue_push(dma_q, \ + dma_make_data(src_spad, psrc0_data), \ + vtcm_layout->src0_spad_half_size, pcur_src0_bytes, pcur_src0_bytes, 1); \ + } \ + } \ + dma_queue_flush(dma_q); \ } #define F32_BYTES(n) ((n) * sizeof(float)) @@ -209,11 +213,13 @@ int op_get_rows(struct htp_ops_context * octx) { if (octx->src[0]->type != HTP_TYPE_F32 && octx->src[0]->type != HTP_TYPE_F16 && - octx->src[0]->type != HTP_TYPE_Q8_0) { + octx->src[0]->type != HTP_TYPE_Q8_0 && + octx->src[0]->type != HTP_TYPE_I32) { return HTP_STATUS_NO_SUPPORT; } - if (octx->dst->type != HTP_TYPE_F32) { + if ((octx->src[0]->type == HTP_TYPE_I32 && octx->dst->type != HTP_TYPE_I32) || + (octx->src[0]->type != HTP_TYPE_I32 && octx->dst->type != HTP_TYPE_F32)) { return HTP_STATUS_NO_SUPPORT; } @@ -221,17 +227,45 @@ int op_get_rows(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + if (htp_tensor_is_extended(octx->src[1])) { + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_tensor * dst = octx->dst; + const uint32_t total_tasks = kparams->total_tasks; + const size_t dst_row_size = htp_tensor_get_row_size(dst->type, dst->ne[0]); + + uint32_t task_start = 0; + uint32_t tasks = total_tasks; + + if (octx->ctx->mdev.count > 1) { + uint32_t tasks_per_chunk = 1; + htp_tensor_mdev_rows_per_chunk(dst, dst_row_size / dst->ne[0], (uint32_t) dst_row_size, &tasks_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, tasks_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + task_start = range.start; + tasks = range.count; + } + + if (tasks == 0) { return HTP_STATUS_OK; } + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + struct get_rows_context grctx; grctx.octx = octx; grctx.kparams = kparams; grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; + grctx.task_start = task_start; + grctx.tasks = tasks; + grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); const uint32_t ne00 = octx->src[0]->ne[0]; - htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, kparams->n_threads); + htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, n_threads); const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); @@ -243,18 +277,19 @@ int op_get_rows(struct htp_ops_context * octx) { case HTP_TYPE_F32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f32_int32_t : get_rows_thread_f32_int64_t); break; case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break; case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break; + case HTP_TYPE_I32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t); break; default: return HTP_STATUS_NO_SUPPORT; } } - FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use_dma=%d n_threads %d\n", + FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use-dma %d n-threads %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - grctx.vtcm_layout.src0_bytes_per_thread * kparams->n_threads, - grctx.vtcm_layout.dst_bytes_per_thread * kparams->n_threads, - kparams->use_dma, kparams->n_threads); + grctx.vtcm_layout.src0_bytes_per_thread * n_threads, + grctx.vtcm_layout.dst_bytes_per_thread * n_threads, + kparams->use_dma, n_threads); - work_queue_run(octx->ctx->work_queue, q_func, &grctx, kparams->n_threads); + work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/hex-common.h b/ggml/src/ggml-hexagon/htp/hex-common.h index 4714486a042f..e6a52540d58b 100644 --- a/ggml/src/ggml-hexagon/htp/hex-common.h +++ b/ggml/src/ggml-hexagon/htp/hex-common.h @@ -77,4 +77,13 @@ static inline bool hex_add_overflow(size_t a, size_t b, size_t *out) { return false; } +static inline uint32_t hex_gcd_u32(uint32_t a, uint32_t b) { + while (b != 0) { + uint32_t t = b; + b = a % b; + a = t; + } + return a; +} + #endif // HEX_COMMON_H diff --git a/ggml/src/ggml-hexagon/htp/hex-dma.h b/ggml/src/ggml-hexagon/htp/hex-dma.h deleted file mode 100644 index 9e9a5f9502a0..000000000000 --- a/ggml/src/ggml-hexagon/htp/hex-dma.h +++ /dev/null @@ -1,2 +0,0 @@ -#pragma once -#include "dma-queue.h" diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.h index 1b3965030009..853f1c1b2d8d 100644 --- a/ggml/src/ggml-hexagon/htp/hex-utils.h +++ b/ggml/src/ggml-hexagon/htp/hex-utils.h @@ -39,7 +39,6 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) { #define HEX_L2_LINE_SIZE 128 #define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration) -#define HEX_L2_FLUSH_IL_THRESHOLD 1024 // inline flush threshold #define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024) #define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) diff --git a/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h b/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h index d6795bf0b57d..d5fb48ad9c61 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +++ b/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h @@ -48,7 +48,7 @@ static const int16_t d_tile_scatter_offsets[64] __attribute__((aligned(128))) = }; // Inner HMX tile computation kernels -static void hmx_fa_qk_dot_tile( +static inline void hmx_fa_qk_dot_tile( const __fp16 * row_tiles, const __fp16 * col_tiles, __fp16 * out_tile, @@ -116,7 +116,7 @@ static void hmx_fa_qk_dot_tile( ); } -static void hmx_fa_o_update_tile( +static inline void hmx_fa_o_update_tile( const __fp16 * d_diag, const __fp16 * o_rc, const __fp16 * p_tile_in, @@ -495,12 +495,140 @@ static inline void hmx_fa_q_prep_fp16( } +// Head-dim-padded Q-prep (f32). Used when DK is not a multiple of 64. +static inline void hmx_fa_q_prep_fp32_pad(__fp16 * vtcm_q_tiles, + const uint8_t * temp_q_vtcm, + size_t start, + size_t end, + size_t g_rows_end, + size_t dk_in, + size_t dk_out, + size_t G, + size_t n_rows_q, + const struct fastdiv_values * div_G, + bool q_transposed) { + const uint32_t n_out_tiles = (uint32_t) (dk_out / 32); + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * dk_out; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < n_out_tiles; ++d) { + ((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_UVector * pv_in0 = (const HVX_UVector *) (temp_q_vtcm + offset0 * dk_in * sizeof(float)); + const HVX_UVector * pv_in1 = (r + 1 < g_rows_end) ? (const HVX_UVector *) (temp_q_vtcm + offset1 * dk_in * sizeof(float)) : NULL; + + for (uint32_t d = 0; d < n_out_tiles; ++d) { + HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS); + const size_t base_lane = (size_t) d * 32; + const size_t real_lanes = (base_lane < dk_in) ? hex_smin(32, dk_in - base_lane) : 0; + + if (real_lanes == 0) { + out_tile[r1 / 2] = Q6_V_vzero(); + continue; + } + + HVX_Vector v0 = pv_in0[d]; + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + if (real_lanes < 32) { + // Straddle tile: keep the first real_lanes floats, zero the padded tail so + // the packed f16 lanes beyond DK are zero. + const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(float))); + v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero()); + } + out_tile[r1 / 2] = hvx_vec_f32_to_f16_shuff(v0, v1); + } + } +} + +// Head-dim-padded Q-prep (f16). Used when DK is not a multiple of 64. +static inline void hmx_fa_q_prep_fp16_pad(__fp16 * vtcm_q_tiles, + const uint8_t * temp_q_vtcm, + size_t start, + size_t end, + size_t g_rows_end, + size_t dk_in, + size_t dk_out, + size_t G, + size_t n_rows_q, + const struct fastdiv_values * div_G, + bool q_transposed) { + const uint32_t n_out_pairs = (uint32_t) (dk_out / 64); + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * dk_out; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < n_out_pairs; ++d) { + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_UVector * pv_in0 = (const HVX_UVector *) (temp_q_vtcm + offset0 * dk_in * sizeof(__fp16)); + const HVX_UVector * pv_in1 = (r + 1 < g_rows_end) ? (const HVX_UVector *) (temp_q_vtcm + offset1 * dk_in * sizeof(__fp16)) : NULL; + + for (uint32_t d = 0; d < n_out_pairs; ++d) { + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + const size_t base_lane = (size_t) d * 64; + const size_t real_lanes = (base_lane < dk_in) ? hex_smin(64, dk_in - base_lane) : 0; + + if (real_lanes == 0) { + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + continue; + } + + HVX_Vector v0 = pv_in0[d]; + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + if (real_lanes < 64) { + const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(__fp16))); + v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero()); + } + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } + } +} + static inline void hmx_fa_q_prep_fallback( __fp16 * vtcm_q_tiles, uintptr_t q_data, size_t q_nb1, size_t q_nb2, size_t q_nb3, uint32_t q_start, uint32_t kv_head, uint32_t ib3, size_t start, size_t end, size_t n_rows_g, - size_t G, size_t DK, bool is_q_fp32, + size_t G, size_t dk_in, size_t dk_out, bool is_q_fp32, const struct fastdiv_values * div_G ) { for (size_t r = start; r < end; r += 2) { @@ -518,33 +646,55 @@ static inline void hmx_fa_q_prep_fallback( size_t r0 = r / HMX_FP16_TILE_N_ROWS; size_t r1 = r % HMX_FP16_TILE_N_ROWS; - __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * dk_out; if (is_q_fp32) { const HVX_UVector * pv_in0 = q_ptr0 ? (const HVX_UVector *) q_ptr0 : NULL; const HVX_UVector * pv_in1 = q_ptr1 ? (const HVX_UVector *) q_ptr1 : NULL; - for (uint32_t d = 0; d < DK / 32; ++d) { - HVX_Vector v0 = pv_in0 ? pv_in0[d] : Q6_V_vzero(); - HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); - HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1); - - HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS); - out_tile[r1 / 2] = v_hf; + for (uint32_t d = 0; d < dk_out / 32; ++d) { + HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS); + const size_t base_lane = (size_t) d * 32; + const size_t real_lanes = (base_lane < dk_in) ? hex_smin(32, dk_in - base_lane) : 0; + + if (real_lanes == 0) { + out_tile[r1 / 2] = Q6_V_vzero(); + continue; + } + HVX_Vector v0 = pv_in0 ? pv_in0[d] : Q6_V_vzero(); + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + if (real_lanes < 32) { + const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(float))); + v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero()); + } + out_tile[r1 / 2] = hvx_vec_f32_to_f16_shuff(v0, v1); } } else { const HVX_UVector * pv_in0 = q_ptr0 ? (const HVX_UVector *) q_ptr0 : NULL; const HVX_UVector * pv_in1 = q_ptr1 ? (const HVX_UVector *) q_ptr1 : NULL; - for (uint32_t d = 0; d < DK / 64; ++d) { - HVX_Vector v0 = pv_in0 ? pv_in0[d] : Q6_V_vzero(); - HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); - HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); - + for (uint32_t d = 0; d < dk_out / 64; ++d) { __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; HVX_Vector * pv_out1 = pv_out0 + 16; + const size_t base_lane = (size_t) d * 64; + const size_t real_lanes = (base_lane < dk_in) ? hex_smin(64, dk_in - base_lane) : 0; + + if (real_lanes == 0) { + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + continue; + } + HVX_Vector v0 = pv_in0 ? pv_in0[d] : Q6_V_vzero(); + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + if (real_lanes < 64) { + const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(__fp16))); + v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero()); + } + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); *pv_out0 = Q6_V_lo_W(vp); *pv_out1 = Q6_V_hi_W(vp); } diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index 0011abba5a8a..d6d40586c5ee 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -506,6 +506,41 @@ static void dequantize_tiled_weight_to_fp16_task_q8_0( } } +// Q6_K stores 6-bit weights and one fp16 scale per 16 k, see HTP_MM_WEIGHT_TILE_SIZE_Q6_K. +// A k-group holds 4 k per row, the HMX tile holds 2, so each group is dealt into two tiles. +static void dequantize_tiled_weight_to_fp16_task_q6_k( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F); + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + const HVX_Vector i32 = Q6_Vb_vsplat_R(32); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size); + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v_sc = vptr[6]; + HVX_Vector v_sc_k16 = Q6_V_vror_VR(v_sc, 64); + HVX_Vector v_scale_k0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2)); + + #pragma unroll + for (int g = 0; g < 8; g++) { + const HVX_Vector v_scale = (g < 4) ? v_scale_k0 : v_scale_k16; + + HVX_Vector v_q = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32); + HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q); + HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4); + + hvx_vmem(dst_ptr + (2 * g + 0) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale)); + hvx_vmem(dst_ptr + (2 * g + 1) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale)); + } + } +} + static __attribute__((noinline)) void convert_f16_weight_to_fp16_tiles_task( const tiled_dequantize_state_t *state, @@ -803,15 +838,12 @@ static void transfer_output_chunk_fp16_to_fp32_col_chunk( HVX_Vector v = ((const HVX_Vector *) tile)[r1]; HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one); - HVX_Vector *pv_out0 = (HVX_Vector *) (output_row_base + c + 0); - HVX_Vector *pv_out1 = (HVX_Vector *) (output_row_base + c + dst_stride); - HVX_Vector v_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); if (src2_row_base) { HVX_Vector v_src2_0 = hvx_vmemu(src2_row_base + c + 0); v_out0 = hvx_vec_add_f32_f32(v_out0, v_src2_0); } - *pv_out0 = v_out0; + hvx_vmemu(output_row_base + c + 0) = v_out0; if (r + 1 < n_rows) { HVX_Vector v_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); @@ -819,7 +851,7 @@ static void transfer_output_chunk_fp16_to_fp32_col_chunk( HVX_Vector v_src2_1 = hvx_vmemu(src2_row_base + c + src2_stride); v_out1 = hvx_vec_add_f32_f32(v_out1, v_src2_1); } - *pv_out1 = v_out1; + hvx_vmemu(output_row_base + c + dst_stride) = v_out1; } } @@ -1366,12 +1398,9 @@ static void transfer_output_chunk_fp16_to_fp32_scattered( HVX_Vector v = ((const HVX_Vector *) tile)[r1]; HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one); - HVX_Vector *pv_out0 = (HVX_Vector *) (output_row0 + c); - HVX_Vector *pv_out1 = output_row1 ? (HVX_Vector *) (output_row1 + c) : NULL; - - *pv_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); - if (pv_out1) { - *pv_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); + hvx_vmemu(output_row0 + c) = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); + if (output_row1) { + hvx_vmemu(output_row1 + c) = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); } } } diff --git a/ggml/src/ggml-hexagon/htp/hmx-utils.h b/ggml/src/ggml-hexagon/htp/hmx-utils.h index 2a61ca7349df..ad295cb7df71 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-utils.h +++ b/ggml/src/ggml-hexagon/htp/hmx-utils.h @@ -27,7 +27,7 @@ static inline void hmx_init_column_scales(void *out_scales, HVX_Vector v_scale) // vscatter offsets for fused dequant+transpose: write K-values directly to [K][N] tile. // word[i] = i*128 maps K-row-pair i to byte offset i*128. // Column offset (n*4) is added at runtime. Entries 0..15 cover one tile (region 2047); -// entries 16..31 cover the next adjacent tile (region 4095) — pick region size at the +// entries 16..31 cover the next adjacent tile (region 4095) - pick region size at the // call site to scatter into one tile (masked) or two contiguous tiles (unmasked). static const int32_t hmx_transpose_scatter_offsets[32] __attribute__((aligned(VLEN))) = { 0 * 128, 1 * 128, 2 * 128, 3 * 128, 4 * 128, 5 * 128, 6 * 128, 7 * 128, 8 * 128, 9 * 128, 10 * 128, @@ -198,16 +198,16 @@ static inline void hmx_interleave_cols_to_tiles(__fp16 * restrict tiles_out, } // --- HMX inline asm macros for load-store packetization --- -#define HMX_LOAD_MPY_F16(act, wt, range) \ - "{\n" \ +#define HMX_LOAD_MPY_F16(act, wt, range) \ + "{\n" \ " activation.hf = mxmem(" act ", " range ")\n" \ - " weight.hf = mxmem(" wt ", " range ")\n" \ + " weight.hf = mxmem(" wt ", " range ")\n" \ "}\n" -#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \ - "{\n" \ +#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \ + "{\n" \ " activation.hf = mxmem(" act ", " range "):deep\n" \ - " weight.hf = mxmem(" wt ", " range ")\n" \ + " weight.hf = mxmem(" wt ", " range ")\n" \ "}\n" #define HMX_STORE_AFTER_F16(out, scale_reg) \ diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index c8a909d61907..814feef7c54d 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -1,7 +1,7 @@ #ifndef HTP_CTX_H #define HTP_CTX_H -#include "hex-dma.h" +#include "dma-queue.h" #include "hmx-queue.h" #include "htp-ops.h" #include "hex-profile.h" @@ -17,16 +17,21 @@ #ifndef HTP_MAX_NTHREADS #define HTP_MAX_NTHREADS 10 #endif -#define HTP_MAX_MMAPS 16 -#define HTP_MAX_DIRTY_RANGES 16 +#define HTP_MAX_MMAPS 64 +#define HTP_MAX_DIRTY_RANGES 64 // Memory mapping struct htp_mmap { uint64_t size; uint64_t base; uint32_t fd; - uint32_t reserved; + uint32_t flags; +}; + +struct htp_dirty_range { + uint32_t start; + uint32_t end; }; // Scratchpad state @@ -38,6 +43,14 @@ struct htp_spad { uint32_t size_per_thread; // size per thread }; +struct htp_mdev_group { + uint16_t idx; + uint16_t count; + struct fastdiv_values count_div; + uint8_t * fence_base; + uint32_t fence_seq; +}; + struct htp_context; // Context while processing an Op @@ -55,9 +68,6 @@ struct htp_ops_context { const struct htp_tensor * dsts[HTP_OP_MAX_OUTPUTS]; }; - dma_queue ** src_dma[HTP_OP_MAX_INPUTS]; - dma_queue ** dst_dma[HTP_OP_MAX_OUTPUTS]; - // TODO convert these to an array struct htp_spad src0_spad; struct htp_spad src1_spad; @@ -65,8 +75,10 @@ struct htp_ops_context { struct htp_spad src3_spad; struct htp_spad dst_spad; - uint32_t n_threads; - uint32_t flags; + uint32_t flags; + uint32_t n_threads; + struct fastdiv_values n_threads_div; + int status; }; // Main context for htp DSP backend @@ -75,7 +87,7 @@ struct htp_context { struct htp_mmap mmap[HTP_MAX_MMAPS]; dma_queue_t dma[HTP_MAX_NTHREADS]; - dma_queue_t dma_cached[HTP_MAX_NTHREADS]; + struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1]; work_queue_t work_queue; hmx_queue_t hmx_queue; @@ -88,7 +100,6 @@ struct htp_context { bool hmx_enabled; bool etm; uint32_t profiler; - struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1]; uint8_t * vtcm_base; size_t vtcm_size; @@ -97,16 +108,13 @@ struct htp_context { atomic_bool vtcm_needs_release; uint64_t max_vmem; - struct htp_dirty_range { - uint32_t start; - uint32_t end; - uint32_t bi; - } dirty_ranges[HTP_MAX_DIRTY_RANGES]; + struct htp_dirty_range dirty_ranges[HTP_MAX_DIRTY_RANGES]; // Persistent DDR scratchpad for MUL_MAT_ID mappings void * ddr_spad_base; size_t ddr_spad_size; + struct htp_mdev_group mdev; struct htp_ops_context octx; qurt_thread_t main_thread; @@ -115,6 +123,27 @@ struct htp_context { size_t footprint; }; +static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) { + if (n_threads == 0 || n_threads > octx->ctx->n_threads) { + return false; + } + + if (n_threads != octx->n_threads) { + octx->n_threads = n_threads; + octx->n_threads_div = n_threads == octx->ctx->n_threads + ? octx->ctx->n_threads_div + : init_fastdiv_values(n_threads); + } + + return true; +} + +static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) { + if (status > HTP_STATUS_OK && octx->status == HTP_STATUS_OK) { + octx->status = status; + } +} + int op_matmul(struct htp_ops_context * octx); int op_matmul_id(struct htp_ops_context * octx); int op_matmul_nx(struct htp_ops_context * octx); @@ -132,6 +161,7 @@ int op_get_rows(struct htp_ops_context * octx); int op_cpy(struct htp_ops_context * octx); int op_repeat(struct htp_ops_context * octx); int op_argsort(struct htp_ops_context * octx); +int op_top_k(struct htp_ops_context * octx); int op_ssm_conv(struct htp_ops_context * octx); int op_cumsum(struct htp_ops_context * octx); int op_fill(struct htp_ops_context * octx); @@ -142,5 +172,6 @@ int op_gated_delta_net(struct htp_ops_context * octx); int op_pad(struct htp_ops_context * octx); int op_im2col(struct htp_ops_context * octx); int op_allreduce(struct htp_ops_context * octx); +int op_roll(struct htp_ops_context * octx); #endif /* HTP_CTX_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp-fence.h b/ggml/src/ggml-hexagon/htp/htp-fence.h new file mode 100644 index 000000000000..7450b5de5363 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-fence.h @@ -0,0 +1,89 @@ +#ifndef HTP_FENCE_H +#define HTP_FENCE_H + +#include <stdatomic.h> +#include <stdint.h> + +#include <HAP_farf.h> + +#include "hex-utils.h" +#include "htp-ops.h" +#include "htp-ctx.h" + +static inline atomic_uint * htp_mdev_fence_slot(const void * fence_base, uint32_t idx) { + return (atomic_uint *) ((const uint8_t *) fence_base + (size_t) idx * HTP_FENCE_SLOT_SIZE); +} + +static inline void htp_fence_write(void * fence_ptr, uint32_t seq, uint32_t status) { + atomic_uint * fence = (atomic_uint *) fence_ptr; + atomic_store(&fence[1], status); + atomic_store(&fence[0], seq); + asm volatile ("syncht" : : : "memory"); + Q6_dccleaninva_A((void *) fence); +} + +static inline void htp_fence_read(const void * fence_ptr, uint32_t * seq, uint32_t * status) { + const atomic_uint * fence = (const atomic_uint *) fence_ptr; + Q6_dccleaninva_A((void *) fence); + asm volatile ("syncht" : : : "memory"); + *seq = atomic_load(&fence[0]); + *status = atomic_load(&fence[1]); +} + +static inline void htp_mdev_group_barrier(struct htp_ops_context * octx) { + struct htp_context * ctx = octx->ctx; + if (ctx->mdev.count <= 1) { + return; + } + + const uint32_t seq = ++ctx->mdev.fence_seq; + + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + + const uint32_t mdev_idx = ctx->mdev.idx; + const uint32_t mdev_count = ctx->mdev.count; + + uint8_t * fence_base = ctx->mdev.fence_base; + atomic_uint * my_fence = htp_mdev_fence_slot(fence_base, mdev_idx); + htp_fence_write(my_fence, seq, octx->status); + + for (uint32_t d = 0; d < mdev_count; d++) { + if (d == mdev_idx) continue; + atomic_uint * peer_fence = htp_mdev_fence_slot(fence_base, d); + uint64_t spins = 0; + while (1) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - seq) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: mdev %u peer %u failed with status %u : seq 0x%08x\n", + mdev_idx, d, peer_status, seq); + htp_ops_context_set_status(octx, peer_status); + } + break; + } + if (++spins == 10000) { + FARF(ALWAYS, "ggml-hex: mdev %u waiting for mdev %u : seq 0x%08x (b %u op %u) my-fence %p peer-fence %p peer-seq 0x%08x (diff %d)\n", + mdev_idx, d, seq, seq >> 12, seq & 0xfff, my_fence, peer_fence, peer_seq, (int32_t)(peer_seq - seq)); + } + if (spins > HTP_FENCE_TIMEOUT) { + FARF(ERROR, "ggml-hex: mdev %u timeout waiting for mdev %u : seq 0x%08x (b %u op %u) peer-fence %p peer-seq 0x%08x\n", + mdev_idx, d, seq, seq >> 12, seq & 0xfff, peer_fence, peer_seq); + htp_ops_context_set_status(octx, HTP_STATUS_INTERNAL_ERR); + break; + } + hex_pause(); + } + } + asm volatile ("syncht" : : : "memory"); + + if (octx->status > HTP_STATUS_OK) { + htp_fence_write(my_fence, seq, octx->status); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); +} + +#endif // HTP_FENCE_H diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index cf938f7eea3a..ee5b924419f2 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -22,6 +22,8 @@ enum htp_data_type { HTP_TYPE_Q4_0 = 2, HTP_TYPE_Q4_1 = 3, HTP_TYPE_Q8_0 = 8, + HTP_TYPE_Q4_K = 12, + HTP_TYPE_Q6_K = 14, HTP_TYPE_IQ4_NL = 20, HTP_TYPE_I32 = 26, HTP_TYPE_I64 = 27, @@ -65,9 +67,11 @@ enum htp_op_code { HTP_OP_UNARY_TANH, HTP_OP_UNARY_ABS, HTP_OP_UNARY_LOG, + HTP_OP_UNARY_RELU, HTP_OP_GLU_SWIGLU, HTP_OP_GLU_SWIGLU_OAI, HTP_OP_GLU_GEGLU, + HTP_OP_GLU_GEGLU_QUICK, HTP_OP_SOFTMAX, HTP_OP_ADD_ID, HTP_OP_ROPE, @@ -76,7 +80,9 @@ enum htp_op_code { HTP_OP_GET_ROWS, HTP_OP_SCALE, HTP_OP_CPY, + HTP_OP_CPY_FENCE, HTP_OP_ARGSORT, + HTP_OP_TOP_K, HTP_OP_SQR, HTP_OP_SQRT, HTP_OP_SUM_ROWS, @@ -93,11 +99,14 @@ enum htp_op_code { HTP_OP_NORM, HTP_OP_CONCAT, HTP_OP_CLAMP, + HTP_OP_LEAKY_RELU, HTP_OP_IM2COL, HTP_OP_FENCE, HTP_OP_ALLREDUCE, HTP_OP_ALLREDUCE_ADD, HTP_OP_GLU_SWIGLU_CLAMP, + HTP_OP_MDEV_GROUP, + HTP_OP_ROLL, HTP_OP_INVALID }; @@ -112,6 +121,7 @@ enum htp_op_code { #define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16) #define HTP_FENCE_TIMEOUT (1000000000ULL) +#define HTP_FENCE_SLOT_SIZE 128 #define HTP_OP_MAX_VMEM_DEFAULT (3355443200u) @@ -123,10 +133,13 @@ enum htp_tensor_flags { HTP_TENSOR_FENCE = (1U << 2) // Tensor is synchronization fence (explicitly managed) }; +enum htp_buf_flags { + HTP_BUF_EXTENDED = (1U << 0), +}; + // Tensor descriptor struct htp_tensor { - uint32_t data; // Buffer offset in the messages, and data pointer on the NPU - uint32_t reserved; // Reserved for alignment padding (must be multiple of 8) + uint64_t data; // Buffer offset in the messages, and data pointer on the NPU uint32_t size; // Data size in bytes uint32_t flags; // Buffer / tensor flags uint32_t type; // Data type @@ -140,12 +153,12 @@ struct htp_tensor { struct htp_buf_desc { uint64_t base; // base address uint64_t size; // total size - uint32_t flags; // buffer flags (unused) + uint32_t flags; // HTP_BUF_* uint32_t fd; // file descriptor }; enum htp_op_flags { - HTP_OPFLAGS_SKIP_COMPUTE = (1U << 0), // Skip actual computation (used for profiling) + HTP_OPFLAGS_STUB = (1U << 0), }; // Op descriptor @@ -191,6 +204,14 @@ enum htp_trace_event_id { HTP_TRACE_EVT_HVX_FA_K_PREP = 29, HTP_TRACE_EVT_HVX_FA_V_PREP = 30, + HTP_TRACE_EVT_HVX_GDN_PREP = 31, + HTP_TRACE_EVT_HVX_GDN_SOLVE = 32, + HTP_TRACE_EVT_HVX_GDN_V_PREP = 33, + HTP_TRACE_EVT_HVX_GDN_D_PREP = 34, + HTP_TRACE_EVT_HVX_GDN_OUT = 35, + HTP_TRACE_EVT_HVX_GDN_STATE = 36, + HTP_TRACE_EVT_HVX_GDN_REM = 37, + HTP_TRACE_EVT_HMX_COMP = 40, }; @@ -212,30 +233,26 @@ struct htp_prof_desc { }; struct htp_opbatch_req { - uint32_t id; // Batch id + uint64_t seq; // Sequence number uint32_t n_bufs; // Number of buffers uint32_t n_tensors; // Number of tensors uint32_t n_ops; // Number of ops uint32_t n_traces; // Number of trace descriptors per thread - uint32_t pad; // unused - uint64_t seq; // Sequence number // struct htp_buf_desc bufs[]; -- dspqueue buf 0 // struct htp_tensor tensors[]; -- dspqueue buf 0 // struct htp_op_desc ops[]; -- dspqueue buf 0 }; struct htp_opbatch_rsp { - uint32_t id; // Batch id - uint32_t status; // HTP_STATUS_... - uint32_t n_bufs; // Number of buffers - uint32_t n_tensors; // Number of tensors - uint32_t n_ops; // Number of op profile descriptors - uint32_t n_traces[HTP_MAX_NTHREADS + 1]; - uint32_t usecs; // Number of usec - uint32_t pad; // align to 8 bytes + uint64_t seq; // Sequence number uint64_t cycles_start; // Start cycle counter uint64_t cycles_stop; // Stop cycle counter - uint64_t seq; // Sequence number + uint32_t status; // HTP_STATUS_... + uint32_t n_bufs; // Number of buffers + uint32_t n_tensors; // Number of tensors + uint32_t n_ops; // Number of op profile descriptors + uint32_t usecs; // Number of usec + uint32_t n_traces[HTP_MAX_NTHREADS + 1]; // struct htp_prof_desc profs[]; -- dspqueue buf 0 }; diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.c b/ggml/src/ggml-hexagon/htp/htp-tensor.c index ae377c9221ff..03b0070d78a7 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.c +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.c @@ -20,7 +20,7 @@ struct l2flush_range { struct l2flush_multi_task { struct htp_thread_trace * trace; - struct l2flush_range ranges[HTP_OP_MAX_INPUTS]; + struct l2flush_range ranges[HTP_MAX_DIRTY_RANGES]; uint32_t n_ranges; uint32_t total_blocks; uint32_t blocks_per_thread; @@ -73,6 +73,27 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); } +static void merge_dirty_ranges(struct htp_context * ctx) { + for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) { + struct htp_dirty_range * r = &ctx->dirty_ranges[i]; + if (!r->start) continue; + + for (uint32_t j = 0; j < HTP_MAX_DIRTY_RANGES;) { + struct htp_dirty_range * s = &ctx->dirty_ranges[j]; + if (i == j || !s->start || r->end < s->start || s->end < r->start) { + j++; + continue; + } + + r->start = MIN(r->start, s->start); + r->end = MAX(r->end, s->end); + s->start = 0; + s->end = 0; + j = 0; + } + } +} + void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { const struct htp_tensor * pending[HTP_OP_MAX_OUTPUTS]; uint32_t n_pending = 0; @@ -83,11 +104,6 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co continue; } - if (t->size <= HEX_L2_FLUSH_IL_THRESHOLD) { - hex_l2flush((void *) (uintptr_t) t->data, t->size); - continue; - } - uint32_t t_start = t->data; uint32_t t_end = t_start + t->size; @@ -110,6 +126,8 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co } } + merge_dirty_ranges(ctx); + if (n_pending == 0) { return; } @@ -132,8 +150,8 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; r->start = pending[i]->data; r->end = pending[i]->data + pending[i]->size; - r->bi = pending[i]->bi; } + merge_dirty_ranges(ctx); return; } @@ -151,12 +169,12 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[i]; r->start = pending[i]->data; r->end = pending[i]->data + pending[i]->size; - r->bi = pending[i]->bi; } + merge_dirty_ranges(ctx); return; } - if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_OP_MAX_INPUTS) { + if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) { struct l2flush_multi_task task; task.trace = ctx->trace; task.n_ranges = n_evict; @@ -195,7 +213,6 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; r->start = pending[i]->data; r->end = pending[i]->data + pending[i]->size; - r->bi = pending[i]->bi; } for (uint32_t i = 0; i < n_empty; i++) { @@ -203,11 +220,16 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; r->start = pending[n_evict + i]->data; r->end = pending[n_evict + i]->data + pending[n_evict + i]->size; - r->bi = pending[n_evict + i]->bi; } + + merge_dirty_ranges(ctx); } static void make_tensor_clean(struct htp_context * ctx, const struct htp_tensor * t) { + if (!t || (t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE))) { + return; + } + uint32_t t_start = t->data; uint32_t t_end = t_start + t->size; @@ -218,6 +240,7 @@ static void make_tensor_clean(struct htp_context * ctx, const struct htp_tensor if (r->start < t_end && t_start < r->end) { if (t_start <= r->start && r->end <= t_end) { r->start = 0; + r->end = 0; } else if (t_start <= r->start) { r->start = t_end; } else if (r->end <= t_end) { @@ -228,6 +251,10 @@ static void make_tensor_clean(struct htp_context * ctx, const struct htp_tensor } static inline bool is_tensor_dirty(struct htp_context * ctx, const struct htp_tensor * t) { + if (!t || (t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE))) { + return false; + } + uint32_t t_start = t->data; uint32_t t_end = t_start + t->size; @@ -242,17 +269,50 @@ static inline bool is_tensor_dirty(struct htp_context * ctx, const struct htp_te return false; } -void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { - const struct htp_tensor * dirty_tensors[HTP_OP_MAX_INPUTS]; - uint32_t n_dirty = 0; +static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_range * ranges, uint32_t n_ranges, uint64_t total_dirty) { + if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { + struct l2flush_multi_task task; + task.trace = ctx->trace; + task.n_ranges = n_ranges; + + uint32_t block_acc = 0; + for (uint32_t i = 0; i < n_ranges; i++) { + const struct htp_dirty_range * r = &ranges[i]; + struct l2flush_range * rg = &task.ranges[i]; + rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE); + rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE); + rg->block_first = block_acc; + rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + block_acc += rg->n_blocks; + } + + task.total_blocks = block_acc; + task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + + work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + } else { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); + for (uint32_t i = 0; i < n_ranges; i++) { + const struct htp_dirty_range * r = &ranges[i]; + hex_l2flush((void *) (uintptr_t) r->start, r->end - r->start); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0); + } +} + +void htp_flush_dirty_ranges(struct htp_context * ctx) { + struct htp_dirty_range ranges[HTP_MAX_DIRTY_RANGES]; + uint32_t n_ranges = 0; uint64_t total_dirty = 0; - for (uint32_t i = 0; i < n; i++) { - const struct htp_tensor * t = tensors[i]; - if (t && !(t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE)) && is_tensor_dirty(ctx, t)) { - dirty_tensors[n_dirty++] = t; - total_dirty += t->size; + for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) { + const struct htp_dirty_range * r = &ctx->dirty_ranges[i]; + if (!r->start) { + continue; } + ranges[n_ranges++] = *r; + total_dirty += r->end - r->start; } if (total_dirty == 0) { @@ -264,37 +324,37 @@ void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * co return; } - if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { - struct l2flush_multi_task task; - task.trace = ctx->trace; - task.n_ranges = 0; + flush_dirty_ranges(ctx, ranges, n_ranges, total_dirty); + memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges)); +} - uint32_t block_acc = 0; - for (uint32_t i = 0; i < n_dirty; i++) { - const struct htp_tensor * t = dirty_tensors[i]; - make_tensor_clean(ctx, t); +void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { + const struct htp_tensor * dirty_tensors[HTP_OP_MAX_INPUTS]; + struct htp_dirty_range ranges[HTP_OP_MAX_INPUTS]; + uint32_t n_dirty = 0; + uint64_t total_dirty = 0; - struct l2flush_range * rg = &task.ranges[task.n_ranges++]; - rg->start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE); - rg->end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE); - rg->block_first = block_acc; - rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; - block_acc += rg->n_blocks; + for (uint32_t i = 0; i < n; i++) { + const struct htp_tensor * t = tensors[i]; + if (is_tensor_dirty(ctx, t)) { + dirty_tensors[n_dirty++] = t; + ranges[n_dirty - 1].start = t->data; + ranges[n_dirty - 1].end = t->data + t->size; + total_dirty += t->size; } + } - task.total_blocks = block_acc; - task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + if (total_dirty == 0) { + return; + } - work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + flush_all_dcache(ctx); return; } - struct htp_thread_trace * tr = &ctx->trace[0]; + flush_dirty_ranges(ctx, ranges, n_dirty, total_dirty); for (uint32_t i = 0; i < n_dirty; i++) { - const struct htp_tensor * t = dirty_tensors[i]; - htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); - hex_l2flush((void *) (uintptr_t) t->data, t->size); - htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); - make_tensor_clean(ctx, t); + make_tensor_clean(ctx, dirty_tensors[i]); } } diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.h b/ggml/src/ggml-hexagon/htp/htp-tensor.h index c9cadbae3f23..f7a96683615a 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.h +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.h @@ -2,27 +2,141 @@ #define HTP_TENSOR_H #include <stdint.h> +#include <stdbool.h> #include "htp-ops.h" #include "hex-bitmap.h" +#include "hex-common.h" +#include "hex-fastdiv.h" + +enum { + HTP_TENSOR_MDEV_LINE_SIZE = 128, +}; + +struct htp_tensor_mdev_range { + uint32_t start; + uint32_t count; +}; static inline void * htp_tensor_data(const struct htp_tensor * t) { return (void *) (uintptr_t) t->data; } +static inline bool htp_tensor_is_extended(const struct htp_tensor * t) { + return t && (t->data >> 32) != 0; +} + static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) { return (uint32_t *) &t->flags; } +static inline bool htp_tensor_is_contiguous(const struct htp_tensor * t, uint32_t type_size) { + uint32_t next_nb = type_size; + if (t->ne[0] != 1 && t->nb[0] != next_nb) { + return false; + } + next_nb *= t->ne[0]; + for (int i = 1; i < HTP_OP_MAX_DIMS; i++) { + if (t->ne[i] != 1 && t->nb[i] != next_nb) { + return false; + } + next_nb *= t->ne[i]; + } + return true; +} + +static inline bool htp_tensor_is_permuted(const struct htp_tensor * t) { + return t->nb[0] > t->nb[1] || t->nb[1] > t->nb[2] || t->nb[2] > t->nb[3]; +} + +static inline bool htp_tensor_mdev_data_aligned(const struct htp_tensor * t) { + return ((uintptr_t) t->data & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0; +} + +static inline bool htp_tensor_can_row_partition(const struct htp_tensor * t, uint32_t elem_size) { + if (!htp_tensor_mdev_data_aligned(t)) { + return false; + } + if (t->ne[0] != 1 && t->nb[0] != elem_size) { + return false; + } + if (htp_tensor_is_permuted(t)) { + return false; + } + if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + if (t->ne[2] > 1 && (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + if (t->ne[3] > 1 && (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + return true; +} + +static inline bool htp_tensor_mdev_rows_per_chunk(const struct htp_tensor * t, uint32_t elem_size, uint32_t row_size, uint32_t * rows_per_chunk) { + *rows_per_chunk = 0; + + if (!htp_tensor_mdev_data_aligned(t)) { + return false; + } + if (t->ne[0] != 1 && t->nb[0] != elem_size) { + return false; + } + if (htp_tensor_is_permuted(t)) { + return false; + } + if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && + (t->ne[2] <= 1 || (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0) && + (t->ne[3] <= 1 || (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) { + *rows_per_chunk = 1; + return true; + } + if (t->nb[1] == row_size && + (t->ne[2] <= 1 || t->nb[2] == t->nb[1] * t->ne[1]) && + (t->ne[3] <= 1 || t->nb[3] == t->nb[2] * t->ne[2])) { + *rows_per_chunk = (row_size > 0) ? (HTP_TENSOR_MDEV_LINE_SIZE / hex_gcd_u32(row_size, HTP_TENSOR_MDEV_LINE_SIZE)) : 1; + return true; + } + return false; +} + +static inline struct htp_tensor_mdev_range htp_tensor_mdev_partition(uint32_t total_units, uint32_t units_per_chunk, uint32_t mdev_idx, uint32_t mdev_count, const struct fastdiv_values * mdev_count_div) { + struct htp_tensor_mdev_range range = { 0, total_units }; + + if (mdev_count <= 1) { + return range; + } + + if (units_per_chunk == 0) { + range.start = (mdev_idx == 0) ? 0 : total_units; + range.count = (mdev_idx == 0) ? total_units : 0; + return range; + } + + const uint32_t total_chunks = total_units / units_per_chunk; + if (total_chunks < mdev_count) { + range.start = (mdev_idx == 0) ? 0 : total_units; + range.count = (mdev_idx == 0) ? total_units : 0; + return range; + } + + const uint32_t chunks_per_mdev = fastdiv(total_chunks + mdev_count - 1, mdev_count_div); + range.start = MIN(mdev_idx * chunks_per_mdev * units_per_chunk, total_units); + if (mdev_idx == mdev_count - 1) { + range.count = total_units - range.start; + } else { + range.count = MIN(chunks_per_mdev * units_per_chunk, total_units - range.start); + } + return range; +} + static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) { switch (type) { case HTP_TYPE_F32: return ne00 * 4; case HTP_TYPE_F16: return ne00 * 2; case HTP_TYPE_Q8_0: return (ne00 / 32) * 34; + case HTP_TYPE_I32: return ne00 * 4; default: return 0; } } struct htp_context; +void htp_flush_dirty_ranges(struct htp_context * ctx); void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); diff --git a/ggml/src/ggml-hexagon/htp/htp_iface.idl b/ggml/src/ggml-hexagon/htp/htp_iface.idl index 47693d8b8b24..b46e252965d9 100644 --- a/ggml/src/ggml-hexagon/htp/htp_iface.idl +++ b/ggml/src/ggml-hexagon/htp/htp_iface.idl @@ -13,7 +13,7 @@ struct htp_iface_pmu_conf { interface htp_iface : remote_handle64 { AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 max_vmem); AEEResult stop(); - AEEResult mmap(in uint32 fd, in uint32 size); + AEEResult mmap(in uint32 fd, in uint64 size); AEEResult munmap(in uint32 fd); AEEResult profiler(in uint32 mode, in htp_iface_pmu_conf pmu); AEEResult etm(in uint32 enable); diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h index 5ef7463426e1..6cbead74c70e 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-arith.h +++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h @@ -16,25 +16,25 @@ #define UNUSED(x) (void)(x) #define hvx_arith_loop_body(dst_type, src0_type, src1_type, elem_size, vec_store, vec_op) \ - do { \ - dst_type * vdst = (dst_type *) dst; \ - src0_type * vsrc0 = (src0_type *) src0; \ - src1_type * vsrc1 = (src1_type *) src1; \ - \ - const uint32_t epv = 128 / (elem_size); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \ - } \ - if (nloe) { \ - HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \ - vec_store((void *) &vdst[i], nloe * (elem_size), v); \ - } \ + do { \ + dst_type * vdst = (dst_type *) dst; \ + src0_type * vsrc0 = (src0_type *) src0; \ + src1_type * vsrc1 = (src1_type *) src1; \ + \ + const uint32_t epv = 128 / (elem_size); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \ + vec_store((void *) &vdst[i], nloe * (elem_size), v); \ + } \ } while(0) #if __HVX_ARCH__ < 79 @@ -56,43 +56,43 @@ #define HVX_OP_MUL_F16(a, b) hvx_vec_mul_f16_f16(a, b) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \ -static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ +#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \ +static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src0 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ +} \ DEFINE_HVX_BINARY_OP_VARIANTS(hvx_add_f32, HVX_OP_ADD_F32, float) DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f32, HVX_OP_SUB_F32, float) @@ -103,25 +103,25 @@ DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f16, HVX_OP_SUB_F16, _Float16) DEFINE_HVX_BINARY_OP_VARIANTS(hvx_mul_f16, HVX_OP_MUL_F16, _Float16) // Dispatcher logic -#define HVX_BINARY_DISPATCHER(OP_NAME) \ +#define HVX_BINARY_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128)) { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ - else OP_NAME##_aau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ - else OP_NAME##_auu(dst, src0, src1, num_elems); \ - } \ - } else { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ - else OP_NAME##_uau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ - else OP_NAME##_uuu(dst, src0, src1, num_elems); \ - } \ - } \ + if (hex_is_aligned((void *) dst, 128)) { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ + else OP_NAME##_aau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ + else OP_NAME##_auu(dst, src0, src1, num_elems); \ + } \ + } else { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ + else OP_NAME##_uau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ + else OP_NAME##_uuu(dst, src0, src1, num_elems); \ + } \ + } \ } HVX_BINARY_DISPATCHER(hvx_add_f32) @@ -166,44 +166,44 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re // Scalar Operations -#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const uint32_t epv = 128 / (elem_size); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - HVX_Vector v = vsrc[i]; \ - vdst[i] = scalar_op_macro(v); \ - } \ - if (nloe) { \ - HVX_Vector v = vsrc[i]; \ - v = scalar_op_macro(v); \ - vec_store((void *) &vdst[i], nloe * (elem_size), v); \ - } \ +#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / (elem_size); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + HVX_Vector v = vsrc[i]; \ + vdst[i] = scalar_op_macro(v); \ + } \ + if (nloe) { \ + HVX_Vector v = vsrc[i]; \ + v = scalar_op_macro(v); \ + vec_store((void *) &vdst[i], nloe * (elem_size), v); \ + } \ } while(0) -#define HVX_OP_ADD_SCALAR_F32(v) \ - ({ \ +#define HVX_OP_ADD_SCALAR_F32(v) \ + ({ \ const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VwVw(inf, v); \ - HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \ - Q6_V_vmux_QVV(pred_inf, inf, out); \ + HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \ + Q6_V_vmux_QVV(pred_inf, inf, out); \ }) #define HVX_OP_MUL_SCALAR_F32(v) HVX_OP_MUL_F32(v, val_vec) #define HVX_OP_SUB_SCALAR_F32(v) HVX_OP_SUB_F32(v, val_vec) -#define HVX_OP_ADD_SCALAR_F16(v) \ - ({ \ +#define HVX_OP_ADD_SCALAR_F16(v) \ + ({ \ const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VhVh(inf, v); \ - HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \ - Q6_V_vmux_QVV(pred_inf, inf, out); \ + HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \ + Q6_V_vmux_QVV(pred_inf, inf, out); \ }) #define HVX_OP_MUL_SCALAR_F16(v) HVX_OP_MUL_F16(v, val_vec) @@ -212,31 +212,31 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re // Scalar Variants // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \ +#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \ static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src % 128 == 0); \ - hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src % 128 == 0); \ + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) dst % 128 == 0); \ - hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) dst % 128 == 0); \ + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) src % 128 == 0); \ - hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) src % 128 == 0); \ + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_add_scalar_f32, HVX_OP_ADD_SCALAR_F32, hvx_vec_splat_f32, float) DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f32, HVX_OP_SUB_SCALAR_F32, hvx_vec_splat_f32, float) @@ -247,17 +247,17 @@ DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f16, HVX_OP_SUB_SCALAR_F16, DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_mul_scalar_f16, HVX_OP_MUL_SCALAR_F16, hvx_vec_splat_f16, _Float16) // Dispatcher logic -#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \ +#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_aa(dst, src, val, num_elems); \ - } else if (hex_is_aligned((void *) dst, 128)) { \ - OP_NAME##_au(dst, src, val, num_elems); \ - } else if (hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_ua(dst, src, val, num_elems); \ - } else { \ - OP_NAME##_uu(dst, src, val, num_elems); \ - } \ + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_aa(dst, src, val, num_elems); \ + } else if (hex_is_aligned((void *) dst, 128)) { \ + OP_NAME##_au(dst, src, val, num_elems); \ + } else if (hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_ua(dst, src, val, num_elems); \ + } else { \ + OP_NAME##_uu(dst, src, val, num_elems); \ + } \ } HVX_BINARY_SCALAR_DISPATCHER(hvx_add_scalar_f32, float) @@ -308,14 +308,54 @@ static inline void hvx_min_scalar_f32(uint8_t * restrict dst, const uint8_t * re } } +// MAX Scalar variants + +#define HVX_OP_MAX_SCALAR(v) Q6_Vsf_vmax_VsfVsf(val_vec, v) + +static inline void hvx_max_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + assert((unsigned long) dst % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float val, const int num_elems) { + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { + hvx_max_scalar_f32_aa(dst, src, val, num_elems); + } else if (hex_is_aligned((void *) dst, 128)) { + hvx_max_scalar_f32_au(dst, src, val, num_elems); + } else if (hex_is_aligned((void *) src, 128)) { + hvx_max_scalar_f32_ua(dst, src, val, num_elems); + } else { + hvx_max_scalar_f32_uu(dst, src, val, num_elems); + } +} + // CLAMP Scalar variants -#define HVX_OP_CLAMP_SCALAR(v) \ - ({ \ +#define HVX_OP_CLAMP_SCALAR(v) \ + ({ \ HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VsfVsf(v, max_vec); \ HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VsfVsf(min_vec, v); \ - HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \ - Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \ + HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \ + Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \ }) static inline void hvx_clamp_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) { @@ -406,6 +446,53 @@ static inline void hvx_clamp_scalar_f16(uint8_t * restrict dst, const uint8_t * } } +#define HVX_OP_LEAKY_RELU_SCALAR(v) \ + ({ \ + HVX_VectorPred pred_neg = Q6_Q_vcmp_gt_VsfVsf(zero_vec, v); \ + HVX_Vector scaled = HVX_OP_MUL_F32(v, ns_vec); \ + Q6_V_vmux_QVV(pred_neg, scaled, v); \ + }) + +static inline void hvx_leaky_relu_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + assert((unsigned long) dst % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, const int num_elems) { + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { + hvx_leaky_relu_scalar_f32_aa(dst, src, ns, num_elems); + } else if (hex_is_aligned((void *) dst, 128)) { + hvx_leaky_relu_scalar_f32_au(dst, src, ns, num_elems); + } else if (hex_is_aligned((void *) src, 128)) { + hvx_leaky_relu_scalar_f32_ua(dst, src, ns, num_elems); + } else { + hvx_leaky_relu_scalar_f32_uu(dst, src, ns, num_elems); + } +} + // // Abs // @@ -627,8 +714,10 @@ static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict #undef HVX_OP_MUL_SCALAR_F16 #undef hvx_scalar_loop_body #undef HVX_OP_MIN_SCALAR +#undef HVX_OP_MAX_SCALAR #undef HVX_OP_CLAMP_SCALAR #undef HVX_OP_CLAMP_SCALAR_F16 +#undef HVX_OP_LEAKY_RELU_SCALAR #undef DEFINE_HVX_BINARY_OP_VARIANTS #undef HVX_BINARY_DISPATCHER #undef UNUSED diff --git a/ggml/src/ggml-hexagon/htp/hvx-div.h b/ggml/src/ggml-hexagon/htp/hvx-div.h index 53ee304e749b..bb7ab0519dae 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-div.h +++ b/ggml/src/ggml-hexagon/htp/hvx-div.h @@ -219,64 +219,64 @@ static inline HVX_Vector hvx_vec_hybrid_div_f16(HVX_Vector vec1, HVX_Vector vec2 } while(0) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ +#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ static inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src0 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \ -} \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \ +} \ // Dispatcher logic -#define HVX_DIV_DISPATCHER(OP_NAME) \ +#define HVX_DIV_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128)) { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ - else OP_NAME##_aau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ - else OP_NAME##_auu(dst, src0, src1, num_elems); \ - } \ - } else { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ - else OP_NAME##_uau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ - else OP_NAME##_uuu(dst, src0, src1, num_elems); \ - } \ - } \ + if (hex_is_aligned((void *) dst, 128)) { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ + else OP_NAME##_aau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ + else OP_NAME##_auu(dst, src0, src1, num_elems); \ + } \ + } else { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ + else OP_NAME##_uau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ + else OP_NAME##_uuu(dst, src0, src1, num_elems); \ + } \ + } \ } DEFINE_HVX_DIV_OP_VARIANTS(hvx_div_f32, hvx_div_f32_loop_body) diff --git a/ggml/src/ggml-hexagon/htp/hvx-exp.h b/ggml/src/ggml-hexagon/htp/hvx-exp.h index bcd3d2d32c61..93ca8cf5133a 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-exp.h +++ b/ggml/src/ggml-hexagon/htp/hvx-exp.h @@ -173,7 +173,7 @@ static inline void hvx_exp_f32(uint8_t * restrict dst, const uint8_t * restrict HVX_Vector * p_vec_in1 = (HVX_Vector *) src; HVX_Vector * p_vec_out = (HVX_Vector *) dst; - #pragma unroll(4) + #pragma unroll(2) for (int i = 0; i < num_elems_whole; i += VLEN_FP32) { if (true == negate) { HVX_Vector neg_vec_in = hvx_vec_neg_f32(*p_vec_in1++); @@ -183,7 +183,7 @@ static inline void hvx_exp_f32(uint8_t * restrict dst, const uint8_t * restrict } } } else { - #pragma unroll(4) + #pragma unroll(2) for (int i = 0; i < num_elems_whole; i += VLEN_FP32) { HVX_Vector in = *(HVX_UVector *) (src + i * SIZEOF_FP32); diff --git a/ggml/src/ggml-hexagon/htp/hvx-inverse.h b/ggml/src/ggml-hexagon/htp/hvx-inverse.h index f2054f45baca..256a8843ba1b 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-inverse.h +++ b/ggml/src/ggml-hexagon/htp/hvx-inverse.h @@ -169,36 +169,36 @@ static inline HVX_Vector hvx_vec_inverse_f16_guard(HVX_Vector v_sf, HVX_Vector n } while(0) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ +#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) src % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \ -} \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \ +} \ // Dispatcher logic -#define HVX_INV_DISPATCHER(OP_NAME) \ +#define HVX_INV_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_aa(dst, src, num_elems); \ - } else if (hex_is_aligned((void *) dst, 128)) { \ - OP_NAME##_au(dst, src, num_elems); \ - } else if (hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_ua(dst, src, num_elems); \ - } else { \ - OP_NAME##_uu(dst, src, num_elems); \ - } \ + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_aa(dst, src, num_elems); \ + } else if (hex_is_aligned((void *) dst, 128)) { \ + OP_NAME##_au(dst, src, num_elems); \ + } else if (hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_ua(dst, src, num_elems); \ + } else { \ + OP_NAME##_uu(dst, src, num_elems); \ + } \ } DEFINE_HVX_INV_OP_VARIANTS(hvx_inverse_f32, hvx_inverse_f32_loop_body) diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h deleted file mode 100644 index 328a8311894a..000000000000 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +++ /dev/null @@ -1,1511 +0,0 @@ -// Dynamic quantizers that produce flat (non-tiled) activations - -static inline void quantize_block_f32_q8_0_flat( - float * restrict x, - uint8_t * restrict y_quants, - __fp16 * restrict y_scales, - uint32_t block_idx -) { - HVX_Vector * vx = (HVX_Vector *) x; - HVX_Vector zero = Q6_V_vzero(); - - HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0])); - HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1])); - HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2])); - HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3])); - - HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); - HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); - HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); - HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); - - HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero); - HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero); - HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero); - HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero); - - HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf))); - HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf))); - - HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); - HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); - - HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); - HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); - - HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); - HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf); - vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); - vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); - - HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf); - HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf); - HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16); - - * (HVX_Vector *) (y_quants + block_idx * 128) = vx_i8; - - HVX_VectorPair vp1 = Q6_W_vshuff_VVR(vd23_hf, vd01_hf, -2); - HVX_VectorPair vp2 = Q6_W_vshuff_VVR(Q6_V_hi_W(vp1), Q6_V_lo_W(vp1), -2); - HVX_Vector v_scales = Q6_V_lo_W(vp2); - hvx_vec_store_u(y_scales + block_idx * 4, 8, v_scales); -} - -static inline void quantize_block_f32_q8_1_flat( - float * restrict x, - uint8_t * restrict y_quants, - __fp16 * restrict y_scales, - uint32_t block_idx -) { - HVX_Vector * vx = (HVX_Vector *) x; - HVX_Vector zero = Q6_V_vzero(); - - HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0])); - HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1])); - HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2])); - HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3])); - - HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); - HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); - HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); - HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); - - HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero); - HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero); - HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero); - HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero); - - HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf))); - HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf))); - - HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); - HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); - - HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); - HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); - - HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); - HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf); - vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); - vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); - - HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf); - HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf); - HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16); - - const HVX_Vector ones = Q6_Vb_vsplat_R(1); - HVX_Vector v_sums = Q6_Vw_vrmpy_VbVb(vx_i8, ones); - v_sums = Q6_Vw_vadd_VwVw(v_sums, Q6_V_vror_VR(v_sums, 4)); - v_sums = Q6_Vw_vadd_VwVw(v_sums, Q6_V_vror_VR(v_sums, 8)); - v_sums = Q6_Vw_vadd_VwVw(v_sums, Q6_V_vror_VR(v_sums, 16)); - - * (HVX_Vector *) (y_quants + block_idx * 128) = vx_i8; - - HVX_VectorPair vp1 = Q6_W_vshuff_VVR(vd23_hf, vd01_hf, -2); - HVX_VectorPair vp2 = Q6_W_vshuff_VVR(Q6_V_hi_W(vp1), Q6_V_lo_W(vp1), -2); - HVX_Vector v_scales = Q6_V_lo_W(vp2); - - HVX_VectorPair v_deal1 = Q6_W_vdeal_VVR(v_sums, v_sums, -4); - HVX_Vector v_even1 = Q6_V_lo_W(v_deal1); - HVX_VectorPair v_deal2 = Q6_W_vdeal_VVR(v_even1, v_even1, -4); - HVX_Vector v_even2 = Q6_V_lo_W(v_deal2); - HVX_VectorPair v_deal3 = Q6_W_vdeal_VVR(v_even2, v_even2, -4); - HVX_Vector v_sums_shuffled = Q6_V_lo_W(v_deal3); - - HVX_Vector v_sums_sf = Q6_Vsf_equals_Vw(v_sums_shuffled); - HVX_Vector v_sums_hf = hvx_vec_f32_to_f16(v_sums_sf, Q6_V_vzero()); - - HVX_Vector v_prod = hvx_vec_mul_f16_f16(v_scales, v_sums_hf); - - HVX_VectorPair vp_scales = Q6_W_vshuff_VVR(v_prod, v_scales, -2); - HVX_Vector v_final = Q6_V_lo_W(vp_scales); - - hvx_vec_store_u(y_scales + block_idx * 8, 16, v_final); -} - -static inline void quantize_row_f32_q8_0_flat(float * restrict x, uint8_t * restrict y, uint32_t k) { - assert(k % 32 == 0); - const uint32_t quants_size = hex_round_up(k, 128); - uint8_t * restrict y_quants = y; - __fp16 * restrict y_scales = (__fp16 *) (y + quants_size); - - const uint32_t nb = (k + 127) / 128; - for (uint32_t i = 0; i < nb; i++) { - quantize_block_f32_q8_0_flat(x + i * 128, y_quants, y_scales, i); - } -} - -static inline void quantize_row_f32_q8_1_flat(float * restrict x, uint8_t * restrict y, uint32_t k) { - assert(k % 32 == 0); - const uint32_t quants_size = hex_round_up(k, 128); - uint8_t * restrict y_quants = y; - __fp16 * restrict y_scales = (__fp16 *) (y + quants_size); - - const uint32_t nb = (k + 127) / 128; - for (uint32_t i = 0; i < nb; i++) { - quantize_block_f32_q8_1_flat(x + i * 128, y_quants, y_scales, i); - } -} - -static inline void quantize_f32_q8_0_flat_kernel( - const uint8_t * restrict src_data, - uint8_t * restrict dst_data, - uint8_t * restrict tmp_data, - uint32_t ne0, - uint32_t nrows, - size_t src_row_size, - size_t dst_row_size -) { - const size_t src_row_size_padded = hex_round_up(src_row_size, QK_Q8_0_TILED * sizeof(float)); - hvx_splat_f32_a(tmp_data, 0.0f, src_row_size_padded / sizeof(float)); - - for (uint32_t i = 0; i < nrows; ++i) { - hex_l2fetch(src_data, src_row_size, src_row_size, 2); - hvx_copy_f32_aa(tmp_data, src_data, ne0); - - quantize_row_f32_q8_0_flat((float *) tmp_data, dst_data, ne0); - dst_data += dst_row_size; - src_data += src_row_size; - } -} - -static inline void quantize_f32_q8_1_flat_kernel( - const uint8_t * restrict src_data, - uint8_t * restrict dst_data, - uint8_t * restrict tmp_data, - uint32_t ne0, - uint32_t nrows, - size_t src_row_size, - size_t dst_row_size -) { - const size_t src_row_size_padded = hex_round_up(src_row_size, QK_Q8_0_TILED * sizeof(float)); - hvx_splat_f32_a(tmp_data, 0.0f, src_row_size_padded / sizeof(float)); - - for (uint32_t i = 0; i < nrows; ++i) { - hex_l2fetch(src_data, src_row_size, src_row_size, 2); - hvx_copy_f32_aa(tmp_data, src_data, ne0); - - quantize_row_f32_q8_1_flat((float *) tmp_data, dst_data, ne0); - dst_data += dst_row_size; - src_data += src_row_size; - } -} - -static inline void quantize_f32_f32_flat_kernel( - const uint8_t * restrict src_data, - uint8_t * restrict dst_data, - uint8_t * restrict tmp_data, - uint32_t ne0, - uint32_t nrows, - size_t src_stride, - size_t dst_stride -) { - (void) tmp_data; - const size_t src_row_size = ne0 * sizeof(float); - for (uint32_t i = 0; i < nrows; ++i) { - hex_l2fetch(src_data, src_row_size, src_stride, 2); - hvx_copy_f32_au(dst_data, src_data, ne0); - - dst_data += dst_stride; - src_data += src_stride; - } -} - -static inline void quantize_f32_f16_flat_kernel( - const uint8_t * restrict src_data, - uint8_t * restrict dst_data, - uint8_t * restrict tmp_data, - uint32_t ne0, - uint32_t nrows, - size_t src_stride, - size_t dst_stride -) { - (void) tmp_data; - const size_t src_row_size = ne0 * sizeof(float); - for (uint32_t i = 0; i < nrows; ++i) { - hex_l2fetch(src_data, src_row_size, src_stride, 2); - hvx_copy_f16_f32_au(dst_data, src_data, ne0); - - dst_data += dst_stride; - src_data += src_stride; - } -} - -static inline void quantize_f16_f16_flat_kernel( - const uint8_t * restrict src_data, - uint8_t * restrict dst_data, - uint8_t * restrict tmp_data, - uint32_t ne0, - uint32_t nrows, - size_t src_stride, - size_t dst_stride -) { - (void) tmp_data; - const size_t src_row_size = ne0 * sizeof(float); - for (uint32_t i = 0; i < nrows; ++i) { - hex_l2fetch(src_data, src_row_size, src_stride, 2); - hvx_copy_f16_au(dst_data, src_data, ne0); - - dst_data += dst_stride; - src_data += src_stride; - } -} - -// Dot kernels that consume flat (non-tiled) activations - -static void flat_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { - const uint8_t * restrict tile_ptr = vx; - const uint8_t * restrict y_q = vy; - - HVX_Vector v_sum_float = Q6_V_vzero(); - HVX_Vector i8 = Q6_Vb_vsplat_R(8); - - static const uint8_t __attribute__((aligned(128))) repl[128] = { - 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - }; - HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; - - const uint32_t quants_size = hex_round_up(n, 128); - const __fp16 * restrict y_scales = (const __fp16 *) (y_q + quants_size); - - uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); - - uint32_t block_idx = kt / 4; - uint32_t sub_idx = kt % 4; - - HVX_Vector vx_i8 = * (const HVX_Vector *) (y_q + block_idx * 128); - HVX_Vector v_act_raw = Q6_V_vror_VR(vx_i8, sub_idx * 32); - - HVX_Vector v_act_rep[8]; - v_act_rep[0] = Q6_V_vdelta_VV(v_act_raw, v_repl_ctrl); - v_act_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 4), v_repl_ctrl); - v_act_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 8), v_repl_ctrl); - v_act_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 12), v_repl_ctrl); - v_act_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 16), v_repl_ctrl); - v_act_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 20), v_repl_ctrl); - v_act_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 24), v_repl_ctrl); - v_act_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 28), v_repl_ctrl); - - HVX_Vector v_sum = accum_4bit_32x1(vptr, v_act_rep, i8); - HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); - - HVX_Vector v_scale_w = vptr[4]; - - __fp16 scale_a_val = y_scales[kt]; - HVX_Vector v_scale_a = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a_val)); - - HVX_Vector v_scale_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a); - HVX_Vector v_sum_scaled = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); - - v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); - } - - if (sz) { - hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); - } else { - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); - } -} - -static void flat_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { - const uint8_t * restrict tile_ptr = vx; - const uint8_t * restrict y0_q = vy0; - const uint8_t * restrict y1_q = vy1; - - HVX_Vector v_sum_float_c0 = Q6_V_vzero(); - HVX_Vector v_sum_float_c1 = Q6_V_vzero(); - HVX_Vector i8 = Q6_Vb_vsplat_R(8); - - static const uint8_t __attribute__((aligned(128))) repl[128] = { - 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - }; - HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; - - const uint32_t quants_size = hex_round_up(n, 128); - const __fp16 * restrict y0_scales = (const __fp16 *) (y0_q + quants_size); - const __fp16 * restrict y1_scales = (const __fp16 *) (y1_q + quants_size); - - uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); - - uint32_t block_idx = kt / 4; - uint32_t sub_idx = kt % 4; - - HVX_Vector vx0_i8 = * (const HVX_Vector *) (y0_q + block_idx * 128); - HVX_Vector vx1_i8 = * (const HVX_Vector *) (y1_q + block_idx * 128); - - HVX_Vector v_act0_raw = Q6_V_vror_VR(vx0_i8, sub_idx * 32); - HVX_Vector v_act1_raw = Q6_V_vror_VR(vx1_i8, sub_idx * 32); - - HVX_Vector v_act0_rep[8]; - v_act0_rep[0] = Q6_V_vdelta_VV(v_act0_raw, v_repl_ctrl); - v_act0_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 4), v_repl_ctrl); - v_act0_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 8), v_repl_ctrl); - v_act0_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 12), v_repl_ctrl); - v_act0_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 16), v_repl_ctrl); - v_act0_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 20), v_repl_ctrl); - v_act0_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 24), v_repl_ctrl); - v_act0_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 28), v_repl_ctrl); - - HVX_Vector v_act1_rep[8]; - v_act1_rep[0] = Q6_V_vdelta_VV(v_act1_raw, v_repl_ctrl); - v_act1_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 4), v_repl_ctrl); - v_act1_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 8), v_repl_ctrl); - v_act1_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 12), v_repl_ctrl); - v_act1_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 16), v_repl_ctrl); - v_act1_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 20), v_repl_ctrl); - v_act1_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 24), v_repl_ctrl); - v_act1_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 28), v_repl_ctrl); - - HVX_VectorPair v_sums = accum_4bit_32x2(vptr, v_act0_rep, v_act1_rep, i8); - HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); - HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); - - HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); - HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); - - HVX_Vector v_scale_w = vptr[4]; - - __fp16 scale_a0_val = y0_scales[kt]; - __fp16 scale_a1_val = y1_scales[kt]; - HVX_Vector v_scale_a0 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a0_val)); - HVX_Vector v_scale_a1 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a1_val)); - - HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a0); - HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a1); - - HVX_Vector v_sum_scaled_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); - HVX_Vector v_sum_scaled_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); - - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); - } - - if (sz0) { - hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); - } else { - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - } - if (sz1) { - hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); - } else { - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); - } -} - -static void flat_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { - const uint8_t * restrict tile_ptr = vx; - const uint8_t * restrict y_q = vy; - - HVX_Vector v_sum_float = Q6_V_vzero(); - - static const uint8_t __attribute__((aligned(128))) repl[128] = { - 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - }; - HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; - - const uint32_t quants_size = hex_round_up(n, 128); - const __fp16 * restrict y_scales = (const __fp16 *) (y_q + quants_size); - - uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); - - uint32_t block_idx = kt / 4; - uint32_t sub_idx = kt % 4; - - HVX_Vector vx_i8 = * (const HVX_Vector *) (y_q + block_idx * 128); - HVX_Vector v_act_raw = Q6_V_vror_VR(vx_i8, sub_idx * 32); - - HVX_Vector v_act_rep[8]; - v_act_rep[0] = Q6_V_vdelta_VV(v_act_raw, v_repl_ctrl); - v_act_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 4), v_repl_ctrl); - v_act_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 8), v_repl_ctrl); - v_act_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 12), v_repl_ctrl); - v_act_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 16), v_repl_ctrl); - v_act_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 20), v_repl_ctrl); - v_act_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 24), v_repl_ctrl); - v_act_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 28), v_repl_ctrl); - - HVX_Vector v_sum = accum_4bit_32x1(vptr, v_act_rep, Q6_V_vzero()); - HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); - - HVX_Vector v_scale_offset = vptr[4]; - HVX_VectorPair p_deal = Q6_W_vdeal_VVR(v_scale_offset, v_scale_offset, -2); - HVX_Vector v_scale = Q6_V_lo_W(p_deal); - HVX_Vector v_offset = Q6_V_hi_W(p_deal); - - __fp16 scale_a_val = y_scales[kt * 2 + 0]; - __fp16 sum_a_val = y_scales[kt * 2 + 1]; - HVX_Vector v_scale_a = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a_val)); - HVX_Vector v_sum_a = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&sum_a_val)); - - HVX_Vector v_scale_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale, v_scale_a); - HVX_Vector v_offset_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset, v_sum_a); - - HVX_Vector v_scaled_dot = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); - HVX_Vector v_sum_scaled = hvx_vec_add_f32_f32(v_scaled_dot, v_offset_comb); - - v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); - } - - if (sz) { - hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); - } else { - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); - } -} - -static void flat_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { - const uint8_t * restrict tile_ptr = vx; - const uint8_t * restrict y0_q = vy0; - const uint8_t * restrict y1_q = vy1; - - HVX_Vector v_sum_float_c0 = Q6_V_vzero(); - HVX_Vector v_sum_float_c1 = Q6_V_vzero(); - - static const uint8_t __attribute__((aligned(128))) repl[128] = { - 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - }; - HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; - - const uint32_t quants_size = hex_round_up(n, 128); - const __fp16 * restrict y0_scales = (const __fp16 *) (y0_q + quants_size); - const __fp16 * restrict y1_scales = (const __fp16 *) (y1_q + quants_size); - - uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); - - uint32_t block_idx = kt / 4; - uint32_t sub_idx = kt % 4; - - HVX_Vector vx0_i8 = * (const HVX_Vector *) (y0_q + block_idx * 128); - HVX_Vector vx1_i8 = * (const HVX_Vector *) (y1_q + block_idx * 128); - - HVX_Vector v_act0_raw = Q6_V_vror_VR(vx0_i8, sub_idx * 32); - HVX_Vector v_act1_raw = Q6_V_vror_VR(vx1_i8, sub_idx * 32); - - HVX_Vector v_act0_rep[8]; - v_act0_rep[0] = Q6_V_vdelta_VV(v_act0_raw, v_repl_ctrl); - v_act0_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 4), v_repl_ctrl); - v_act0_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 8), v_repl_ctrl); - v_act0_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 12), v_repl_ctrl); - v_act0_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 16), v_repl_ctrl); - v_act0_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 20), v_repl_ctrl); - v_act0_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 24), v_repl_ctrl); - v_act0_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 28), v_repl_ctrl); - - HVX_Vector v_act1_rep[8]; - v_act1_rep[0] = Q6_V_vdelta_VV(v_act1_raw, v_repl_ctrl); - v_act1_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 4), v_repl_ctrl); - v_act1_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 8), v_repl_ctrl); - v_act1_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 12), v_repl_ctrl); - v_act1_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 16), v_repl_ctrl); - v_act1_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 20), v_repl_ctrl); - v_act1_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 24), v_repl_ctrl); - v_act1_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 28), v_repl_ctrl); - - HVX_VectorPair v_sums = accum_4bit_32x2(vptr, v_act0_rep, v_act1_rep, Q6_V_vzero()); - HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); - HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); - - HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); - HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); - - HVX_Vector v_scale_offset = vptr[4]; - HVX_VectorPair p_deal = Q6_W_vdeal_VVR(v_scale_offset, v_scale_offset, -2); - HVX_Vector v_scale = Q6_V_lo_W(p_deal); - HVX_Vector v_offset = Q6_V_hi_W(p_deal); - - __fp16 scale_a0_val = y0_scales[kt * 2 + 0]; - __fp16 sum_a0_val = y0_scales[kt * 2 + 1]; - __fp16 scale_a1_val = y1_scales[kt * 2 + 0]; - __fp16 sum_a1_val = y1_scales[kt * 2 + 1]; - - HVX_Vector v_scale_a0 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a0_val)); - HVX_Vector v_sum_a0 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&sum_a0_val)); - HVX_Vector v_scale_a1 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a1_val)); - HVX_Vector v_sum_a1 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&sum_a1_val)); - - HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale, v_scale_a0); - HVX_Vector v_offset_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset, v_sum_a0); - HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale, v_scale_a1); - HVX_Vector v_offset_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset, v_sum_a1); - - HVX_Vector v_scaled_dot_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); - HVX_Vector v_sum_scaled_c0 = hvx_vec_add_f32_f32(v_scaled_dot_c0, v_offset_comb_c0); - - HVX_Vector v_scaled_dot_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); - HVX_Vector v_sum_scaled_c1 = hvx_vec_add_f32_f32(v_scaled_dot_c1, v_offset_comb_c1); - - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); - } - - if (sz0) { - hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); - } else { - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - } - if (sz1) { - hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); - } else { - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); - } -} - -static void flat_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { - const uint8_t * restrict tile_ptr = vx; - const uint8_t * restrict y_q = vy; - - HVX_Vector v_sum_float = Q6_V_vzero(); - - static const uint8_t __attribute__((aligned(128))) repl[128] = { - 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - }; - HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; - - const uint32_t quants_size = hex_round_up(n, 128); - const __fp16 * restrict y_scales = (const __fp16 *) (y_q + quants_size); - - uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 1152); - - uint32_t block_idx = kt / 4; - uint32_t sub_idx = kt % 4; - - HVX_Vector vx_i8 = * (const HVX_Vector *) (y_q + block_idx * 128); - HVX_Vector v_act_raw = Q6_V_vror_VR(vx_i8, sub_idx * 32); - - HVX_Vector v_act_rep[8]; - v_act_rep[0] = Q6_V_vdelta_VV(v_act_raw, v_repl_ctrl); - v_act_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 4), v_repl_ctrl); - v_act_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 8), v_repl_ctrl); - v_act_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 12), v_repl_ctrl); - v_act_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 16), v_repl_ctrl); - v_act_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 20), v_repl_ctrl); - v_act_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 24), v_repl_ctrl); - v_act_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 28), v_repl_ctrl); - - HVX_Vector v_sum = accum_q8_0_32x1(vptr, v_act_rep); - HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); - - HVX_Vector v_scale_w = vptr[8]; - - __fp16 scale_a_val = y_scales[kt]; - HVX_Vector v_scale_a = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a_val)); - - HVX_Vector v_scale_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a); - HVX_Vector v_sum_scaled = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); - - v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); - } - - if (sz) { - hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); - } else { - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); - } -} - -static void flat_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { - const uint8_t * restrict tile_ptr = vx; - const uint8_t * restrict y0_q = vy0; - const uint8_t * restrict y1_q = vy1; - - HVX_Vector v_sum_float_c0 = Q6_V_vzero(); - HVX_Vector v_sum_float_c1 = Q6_V_vzero(); - - static const uint8_t __attribute__((aligned(128))) repl[128] = { - 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - }; - HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; - - const uint32_t quants_size = hex_round_up(n, 128); - const __fp16 * restrict y0_scales = (const __fp16 *) (y0_q + quants_size); - const __fp16 * restrict y1_scales = (const __fp16 *) (y1_q + quants_size); - - uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 1152); - - uint32_t block_idx = kt / 4; - uint32_t sub_idx = kt % 4; - - HVX_Vector vx0_i8 = * (const HVX_Vector *) (y0_q + block_idx * 128); - HVX_Vector vx1_i8 = * (const HVX_Vector *) (y1_q + block_idx * 128); - - HVX_Vector v_act0_raw = Q6_V_vror_VR(vx0_i8, sub_idx * 32); - HVX_Vector v_act1_raw = Q6_V_vror_VR(vx1_i8, sub_idx * 32); - - HVX_Vector v_act0_rep[8]; - v_act0_rep[0] = Q6_V_vdelta_VV(v_act0_raw, v_repl_ctrl); - v_act0_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 4), v_repl_ctrl); - v_act0_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 8), v_repl_ctrl); - v_act0_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 12), v_repl_ctrl); - v_act0_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 16), v_repl_ctrl); - v_act0_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 20), v_repl_ctrl); - v_act0_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 24), v_repl_ctrl); - v_act0_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 28), v_repl_ctrl); - - HVX_Vector v_act1_rep[8]; - v_act1_rep[0] = Q6_V_vdelta_VV(v_act1_raw, v_repl_ctrl); - v_act1_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 4), v_repl_ctrl); - v_act1_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 8), v_repl_ctrl); - v_act1_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 12), v_repl_ctrl); - v_act1_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 16), v_repl_ctrl); - v_act1_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 20), v_repl_ctrl); - v_act1_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 24), v_repl_ctrl); - v_act1_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 28), v_repl_ctrl); - - HVX_VectorPair v_sums = accum_q8_0_32x2(vptr, v_act0_rep, v_act1_rep); - HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); - HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); - - HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); - HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); - - HVX_Vector v_scale_w = vptr[8]; - - __fp16 scale_a0_val = y0_scales[kt]; - __fp16 scale_a1_val = y1_scales[kt]; - HVX_Vector v_scale_a0 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a0_val)); - HVX_Vector v_scale_a1 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a1_val)); - - HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a0); - HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a1); - - HVX_Vector v_sum_scaled_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); - HVX_Vector v_sum_scaled_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); - - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); - } - - if (sz0) { - hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); - } else { - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - } - if (sz1) { - hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); - } else { - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); - } -} - -static void flat_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { - const uint8_t * restrict tile_ptr = vx; - const uint8_t * restrict y_q = vy; - - HVX_Vector v_sum_float = Q6_V_vzero(); - HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - HVX_Vector lut = *(const HVX_Vector *) kvalues_iq4nl_lut; - - static const uint8_t __attribute__((aligned(128))) repl[128] = { - 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - }; - HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; - - const uint32_t quants_size = hex_round_up(n, 128); - const __fp16 * restrict y_scales = (const __fp16 *) (y_q + quants_size); - - uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); - - uint32_t block_idx = kt / 4; - uint32_t sub_idx = kt % 4; - - HVX_Vector vx = * (const HVX_Vector *) (y_q + block_idx * 128); - HVX_Vector v_act_raw = Q6_V_vror_VR(vx, sub_idx * 32); - - HVX_Vector v_act_rep[8]; - v_act_rep[0] = Q6_V_vdelta_VV(v_act_raw, v_repl_ctrl); - v_act_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 4), v_repl_ctrl); - v_act_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 8), v_repl_ctrl); - v_act_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 12), v_repl_ctrl); - v_act_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 16), v_repl_ctrl); - v_act_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 20), v_repl_ctrl); - v_act_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 24), v_repl_ctrl); - v_act_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 28), v_repl_ctrl); - - HVX_Vector v_sum = accum_4bit_32x1_lut(vptr, v_act_rep, mask_h4, lut); - HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); - - HVX_Vector v_scale_w = vptr[4]; - - __fp16 scale_a_val = y_scales[kt]; - HVX_Vector v_scale_a = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a_val)); - - HVX_Vector v_scale_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a); - HVX_Vector v_sum_scaled = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); - - v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); - } - - if (sz) { - hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); - } else { - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); - } -} - -static void flat_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { - const uint8_t * restrict tile_ptr = vx; - const uint8_t * restrict y0_q = vy0; - const uint8_t * restrict y1_q = vy1; - - HVX_Vector v_sum_float_c0 = Q6_V_vzero(); - HVX_Vector v_sum_float_c1 = Q6_V_vzero(); - HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - HVX_Vector lut = *(const HVX_Vector *) kvalues_iq4nl_lut; - - static const uint8_t __attribute__((aligned(128))) repl[128] = { - 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - }; - HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; - - const uint32_t quants_size = hex_round_up(n, 128); - const __fp16 * restrict y0_scales = (const __fp16 *) (y0_q + quants_size); - const __fp16 * restrict y1_scales = (const __fp16 *) (y1_q + quants_size); - - uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); - - uint32_t block_idx = kt / 4; - uint32_t sub_idx = kt % 4; - - HVX_Vector vx0 = * (const HVX_Vector *) (y0_q + block_idx * 128); - HVX_Vector vx1 = * (const HVX_Vector *) (y1_q + block_idx * 128); - - HVX_Vector v_act0_raw = Q6_V_vror_VR(vx0, sub_idx * 32); - HVX_Vector v_act1_raw = Q6_V_vror_VR(vx1, sub_idx * 32); - - HVX_Vector v_act0_rep[8]; - v_act0_rep[0] = Q6_V_vdelta_VV(v_act0_raw, v_repl_ctrl); - v_act0_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 4), v_repl_ctrl); - v_act0_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 8), v_repl_ctrl); - v_act0_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 12), v_repl_ctrl); - v_act0_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 16), v_repl_ctrl); - v_act0_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 20), v_repl_ctrl); - v_act0_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 24), v_repl_ctrl); - v_act0_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 28), v_repl_ctrl); - - HVX_Vector v_act1_rep[8]; - v_act1_rep[0] = Q6_V_vdelta_VV(v_act1_raw, v_repl_ctrl); - v_act1_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 4), v_repl_ctrl); - v_act1_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 8), v_repl_ctrl); - v_act1_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 12), v_repl_ctrl); - v_act1_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 16), v_repl_ctrl); - v_act1_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 20), v_repl_ctrl); - v_act1_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 24), v_repl_ctrl); - v_act1_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 28), v_repl_ctrl); - - HVX_VectorPair v_sums = accum_4bit_32x2_lut(vptr, v_act0_rep, v_act1_rep, mask_h4, lut); - HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); - HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); - - HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); - HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); - - HVX_Vector v_scale_w = vptr[4]; - - __fp16 scale_a0_val = y0_scales[kt]; - __fp16 scale_a1_val = y1_scales[kt]; - HVX_Vector v_scale_a0 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a0_val)); - HVX_Vector v_scale_a1 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a1_val)); - - HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a0); - HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a1); - - HVX_Vector v_sum_scaled_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); - HVX_Vector v_sum_scaled_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); - - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); - } - - if (sz0) { - hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); - } else { - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - } - if (sz1) { - hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); - } else { - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); - } -} - -static void flat_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { - const uint8_t * restrict tile_ptr = vx; - const uint8_t * restrict y_q = vy; - - HVX_Vector v_sum_float = Q6_V_vzero(); - HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - HVX_Vector lut = *(const HVX_Vector *) kvalues_mxfp4_lut; - HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0; - HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff); - - static const uint8_t __attribute__((aligned(128))) repl[128] = { - 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - }; - HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; - - const uint32_t quants_size = hex_round_up(n, 128); - const __fp16 * restrict y_scales = (const __fp16 *) (y_q + quants_size); - - uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); - - uint32_t block_idx = kt / 4; - uint32_t sub_idx = kt % 4; - - HVX_Vector vx = * (const HVX_Vector *) (y_q + block_idx * 128); - HVX_Vector v_act_raw = Q6_V_vror_VR(vx, sub_idx * 32); - - HVX_Vector v_act_rep[8]; - v_act_rep[0] = Q6_V_vdelta_VV(v_act_raw, v_repl_ctrl); - v_act_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 4), v_repl_ctrl); - v_act_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 8), v_repl_ctrl); - v_act_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 12), v_repl_ctrl); - v_act_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 16), v_repl_ctrl); - v_act_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 20), v_repl_ctrl); - v_act_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 24), v_repl_ctrl); - v_act_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 28), v_repl_ctrl); - - HVX_Vector v_sum = accum_4bit_32x1_lut(vptr, v_act_rep, mask_h4, lut); - HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); - - HVX_Vector v_scale_w = hvx_vmem(tile_ptr + kt * 640 + 512); - HVX_Vector r0_d = Q6_V_vdelta_VV(v_scale_w, expand); - r0_d = Q6_V_vand_VV(r0_d, e8m0_mask); - HVX_Vector v_scale_w_f32 = Q6_Vw_vasl_VwR(r0_d, 23); - - __fp16 scale_a_val = y_scales[kt]; - HVX_Vector v_scale_a_f16 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a_val)); - HVX_VectorPair p_scale_a_f32 = hvx_vec_f16_to_f32(v_scale_a_f16); - HVX_Vector v_scale_a = Q6_V_lo_W(p_scale_a_f32); - - HVX_Vector v_scale_comb = hvx_vec_mul_f32_f32(v_scale_w_f32, v_scale_a); - HVX_Vector v_sum_scaled = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); - - v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); - } - - v_sum_float = hvx_vec_mul_f32_f32(v_sum_float, hvx_vec_splat_f32(0.5f)); - - if (sz) { - hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); - } else { - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); - } -} - -static void flat_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { - const uint8_t * restrict tile_ptr = vx; - const uint8_t * restrict y0_q = vy0; - const uint8_t * restrict y1_q = vy1; - - HVX_Vector v_sum_float_c0 = Q6_V_vzero(); - HVX_Vector v_sum_float_c1 = Q6_V_vzero(); - HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - HVX_Vector lut = *(const HVX_Vector *) kvalues_mxfp4_lut; - HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0; - HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff); - - static const uint8_t __attribute__((aligned(128))) repl[128] = { - 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, - }; - HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; - - const uint32_t quants_size = hex_round_up(n, 128); - const __fp16 * restrict y0_scales = (const __fp16 *) (y0_q + quants_size); - const __fp16 * restrict y1_scales = (const __fp16 *) (y1_q + quants_size); - - uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); - - uint32_t block_idx = kt / 4; - uint32_t sub_idx = kt % 4; - - HVX_Vector vx0 = * (const HVX_Vector *) (y0_q + block_idx * 128); - HVX_Vector vx1 = * (const HVX_Vector *) (y1_q + block_idx * 128); - - HVX_Vector v_act0_raw = Q6_V_vror_VR(vx0, sub_idx * 32); - HVX_Vector v_act1_raw = Q6_V_vror_VR(vx1, sub_idx * 32); - - HVX_Vector v_act0_rep[8]; - v_act0_rep[0] = Q6_V_vdelta_VV(v_act0_raw, v_repl_ctrl); - v_act0_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 4), v_repl_ctrl); - v_act0_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 8), v_repl_ctrl); - v_act0_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 12), v_repl_ctrl); - v_act0_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 16), v_repl_ctrl); - v_act0_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 20), v_repl_ctrl); - v_act0_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 24), v_repl_ctrl); - v_act0_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 28), v_repl_ctrl); - - HVX_Vector v_act1_rep[8]; - v_act1_rep[0] = Q6_V_vdelta_VV(v_act1_raw, v_repl_ctrl); - v_act1_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 4), v_repl_ctrl); - v_act1_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 8), v_repl_ctrl); - v_act1_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 12), v_repl_ctrl); - v_act1_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 16), v_repl_ctrl); - v_act1_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 20), v_repl_ctrl); - v_act1_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 24), v_repl_ctrl); - v_act1_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 28), v_repl_ctrl); - - HVX_VectorPair v_sums = accum_4bit_32x2_lut(vptr, v_act0_rep, v_act1_rep, mask_h4, lut); - HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); - HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); - - HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); - HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); - - HVX_Vector v_scale_w = hvx_vmem(tile_ptr + kt * 640 + 512); - HVX_Vector r0_d = Q6_V_vdelta_VV(v_scale_w, expand); - r0_d = Q6_V_vand_VV(r0_d, e8m0_mask); - HVX_Vector v_scale_w_f32 = Q6_Vw_vasl_VwR(r0_d, 23); - - __fp16 scale_a0_val = y0_scales[kt]; - __fp16 scale_a1_val = y1_scales[kt]; - HVX_Vector v_scale_a0_f16 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a0_val)); - HVX_Vector v_scale_a1_f16 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a1_val)); - HVX_VectorPair p_scale_a0_f32 = hvx_vec_f16_to_f32(v_scale_a0_f16); - HVX_VectorPair p_scale_a1_f32 = hvx_vec_f16_to_f32(v_scale_a1_f16); - HVX_Vector v_scale_a0 = Q6_V_lo_W(p_scale_a0_f32); - HVX_Vector v_scale_a1 = Q6_V_lo_W(p_scale_a1_f32); - - HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f32_f32(v_scale_w_f32, v_scale_a0); - HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f32_f32(v_scale_w_f32, v_scale_a1); - - HVX_Vector v_sum_scaled_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); - HVX_Vector v_sum_scaled_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); - - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); - } - - v_sum_float_c0 = hvx_vec_mul_f32_f32(v_sum_float_c0, hvx_vec_splat_f32(0.5f)); - v_sum_float_c1 = hvx_vec_mul_f32_f32(v_sum_float_c1, hvx_vec_splat_f32(0.5f)); - - if (sz0) { - hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); - } else { - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - } - if (sz1) { - hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); - } else { - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); - } -} - -#if __HVX_ARCH__ < 79 -#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b)) -#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b)) -#else -#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b) -#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b) -#endif - -static inline void vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { - const HVX_Vector * restrict x = (const HVX_Vector *) vx; - const HVX_Vector * restrict y = (const HVX_Vector *) vy; - - uint32_t nvec = n / VLEN_FP32; // num full fp32 hvx vectors - uint32_t nloe = n % VLEN_FP32; // leftover elements - - HVX_Vector rsum = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(4) - for (i = 0; i < nvec; i++) { - HVX_Vector prod = HVX_OP_MUL_F32(x[i], y[i]); - rsum = HVX_OP_ADD_F32(rsum, prod); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); - HVX_Vector x_sf = Q6_V_vand_QV(bmask, x[i]); - HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); - HVX_Vector prod = HVX_OP_MUL_F32(x_sf, y_sf); - rsum = HVX_OP_ADD_F32(rsum, prod); - } - - *s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum)); -} - -static inline void vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0) { - const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; - const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; - const HVX_Vector * restrict y = (const HVX_Vector *) vy0; - - uint32_t nvec = n / VLEN_FP32; - uint32_t nloe = n % VLEN_FP32; - - HVX_Vector rsum0 = Q6_V_vzero(); - HVX_Vector rsum1 = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector y_sf = y[i]; - HVX_Vector prod0 = HVX_OP_MUL_F32(x0[i], y_sf); - HVX_Vector prod1 = HVX_OP_MUL_F32(x1[i], y_sf); - rsum0 = HVX_OP_ADD_F32(rsum0, prod0); - rsum1 = HVX_OP_ADD_F32(rsum1, prod1); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); - HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); - HVX_Vector x0_sf = Q6_V_vand_QV(bmask, x0[i]); - HVX_Vector x1_sf = Q6_V_vand_QV(bmask, x1[i]); - HVX_Vector prod0 = HVX_OP_MUL_F32(x0_sf, y_sf); - HVX_Vector prod1 = HVX_OP_MUL_F32(x1_sf, y_sf); - rsum0 = HVX_OP_ADD_F32(rsum0, prod0); - rsum1 = HVX_OP_ADD_F32(rsum1, prod1); - } - - HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); - hvx_vec_store_u(s0, 8, rsum); -} - -static inline void vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0, const void * restrict vy1) { - const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; - const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; - const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; - const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; - - uint32_t nvec = n / VLEN_FP32; - uint32_t nloe = n % VLEN_FP32; - - HVX_Vector r0_c0_sum = Q6_V_vzero(); - HVX_Vector r0_c1_sum = Q6_V_vzero(); - HVX_Vector r1_c0_sum = Q6_V_vzero(); - HVX_Vector r1_c1_sum = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector r0_sf = x0[i]; - HVX_Vector r1_sf = x1[i]; - HVX_Vector c0_sf = y0[i]; - HVX_Vector c1_sf = y1[i]; - - r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); - r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); - r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); - r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); - - HVX_Vector r0_sf = Q6_V_vand_QV(bmask, x0[i]); - HVX_Vector r1_sf = Q6_V_vand_QV(bmask, x1[i]); - HVX_Vector c0_sf = Q6_V_vand_QV(bmask, y0[i]); - HVX_Vector c1_sf = Q6_V_vand_QV(bmask, y1[i]); - - r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); - r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); - r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); - r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); - } - - // Reduce and store results - HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); - HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); - - hvx_vec_store_u(s0, 8, r0_r1_c0_sum); - hvx_vec_store_u(s1, 8, r0_r1_c1_sum); -} - -static inline void vec_dot_f32_f32_uu_1x1(const uint32_t n, float * restrict s, const void * restrict x, const void * restrict y) { - const HVX_UVector * restrict vx = (const HVX_UVector * restrict) x; - const HVX_UVector * restrict vy = (const HVX_UVector * restrict) y; - - uint32_t nvec = n / VLEN_FP32; // num full fp32 hvx vectors - uint32_t nloe = n % VLEN_FP32; // leftover elements - - HVX_Vector rsum = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector x_sf = vx[i]; - HVX_Vector y_sf = vy[i]; - - rsum = HVX_OP_ADD_F32(rsum, HVX_OP_MUL_F32(x_sf, y_sf)); - } - - if (nloe) { - HVX_Vector x_sf = vx[i]; - HVX_Vector y_sf = vy[i]; - - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); - x_sf = Q6_V_vand_QV(bmask, x_sf); - y_sf = Q6_V_vand_QV(bmask, y_sf); - - rsum = HVX_OP_ADD_F32(rsum, HVX_OP_MUL_F32(x_sf, y_sf)); - } - - rsum = hvx_vec_reduce_sum_f32(rsum); - hvx_vec_store_u(&s[0], 4, rsum); -} - -#undef HVX_OP_ADD_F32 -#undef HVX_OP_MUL_F32 - -static inline void vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { - const HVX_Vector * restrict x = (const HVX_Vector *) vx; - const HVX_Vector * restrict y = (const HVX_Vector *) vy; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - HVX_VectorPair rsum_p = Q6_W_vzero(); - - uint32_t i = 0; - - #pragma unroll(4) - for (i = 0; i < nvec; i++) { - rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x[i], y[i]); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); - rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf); - } - - HVX_Vector rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p))); - hvx_vec_store_u(s, 4, hvx_vec_reduce_sum_f32(rsum)); -} - -static inline void vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0) { - const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; - const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; - const HVX_Vector * restrict y = (const HVX_Vector *) vy0; - - uint32_t nvec = n / VLEN_FP16; - uint32_t nloe = n % VLEN_FP16; - - HVX_VectorPair rsum0_p = Q6_W_vzero(); - HVX_VectorPair rsum1_p = Q6_W_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector y_hf = y[i]; - rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0[i], y_hf); - rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1[i], y_hf); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); - HVX_Vector x0_hf = Q6_V_vand_QV(bmask, x0[i]); - HVX_Vector x1_hf = Q6_V_vand_QV(bmask, x1[i]); - rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); - rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); - } - - HVX_Vector rsum0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p))); - HVX_Vector rsum1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p))); - HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); - hvx_vec_store_u(s0, 8, rsum); -} - -static inline void vec_dot_f16_f16_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0, const void * restrict vy1) { - const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; - const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; - const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; - const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; - - uint32_t nvec = n / VLEN_FP16; - uint32_t nloe = n % VLEN_FP16; - - // Row sums (sf) - 4 accumulators for 2x2 tile - HVX_VectorPair r0_c0_sum_p = Q6_W_vzero(); - HVX_VectorPair r0_c1_sum_p = Q6_W_vzero(); - HVX_VectorPair r1_c0_sum_p = Q6_W_vzero(); - HVX_VectorPair r1_c1_sum_p = Q6_W_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector r0_hf = x0[i]; - HVX_Vector r1_hf = x1[i]; - HVX_Vector c0_hf = y0[i]; - HVX_Vector c1_hf = y1[i]; - - // Compute 4 dot products: r0xc0, r0xc1, r1xc0, r1xc1 - r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); - r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); - r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); - r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - - HVX_Vector r0_hf = Q6_V_vand_QV(bmask, x0[i]); - HVX_Vector r1_hf = Q6_V_vand_QV(bmask, x1[i]); - HVX_Vector c0_hf = Q6_V_vand_QV(bmask, y0[i]); - HVX_Vector c1_hf = Q6_V_vand_QV(bmask, y1[i]); - - r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); - r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); - r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); - r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); - } - - HVX_Vector r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c0_sum_p), Q6_V_hi_W(r0_c0_sum_p))); - HVX_Vector r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c1_sum_p), Q6_V_hi_W(r0_c1_sum_p))); - HVX_Vector r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c0_sum_p), Q6_V_hi_W(r1_c0_sum_p))); - HVX_Vector r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c1_sum_p), Q6_V_hi_W(r1_c1_sum_p))); - - // Reduce and store results - HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); - HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); - - hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0 - hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1 -} - -static inline void vec_dot_f16_f16_uu_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { - const HVX_UVector * restrict x = (const HVX_UVector *) vx; - const HVX_UVector * restrict y = (const HVX_UVector *) vy; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - HVX_Vector rsum = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(4) - for (i = 0; i < nvec; i++) { - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x[i], y[i]); - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); - - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); - } - - rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum)); - hvx_vec_store_u(&s[0], 4, rsum); -} - -static inline void vec_dot_f16_f32_uu_1x1(const uint32_t n, float * restrict s, const void * restrict x, const void * restrict y) { - const HVX_UVector * restrict vx = (const HVX_UVector * restrict) x; - const HVX_UVector * restrict vy = (const HVX_UVector * restrict) y; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - const HVX_Vector zero = Q6_V_vzero(); - - HVX_Vector rsum = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - // Load y (fp32) and convert into fp16 - HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements - HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements - HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf))); - - // Load x (fp16) - HVX_Vector x_hf = vx[i]; - - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); - - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); - } - - if (nloe) { - // Load y (fp32) and convert into fp16 - HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements - HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements - HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf))); - - // Load x (fp16) - HVX_Vector x_hf = vx[i]; - - // Zero-out unused elements - // Note that we need to clear both x and y because they may contain NANs - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - x_hf = Q6_V_vand_QV(bmask, x_hf); - y_hf = Q6_V_vand_QV(bmask, y_hf); - - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); - - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); - } - - // Convert into fp32 and reduce - rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum)); - hvx_vec_store_u(&s[0], 4, rsum); -} - -static inline void hvx_tensor_add_f32_grid( - const struct htp_tensor * restrict dst, - const struct htp_tensor * restrict src2, - uint32_t start_row, - uint32_t end_row, - uint32_t start_col, - uint32_t end_col, - const struct fastdiv_values * div_ne11_12, - const struct fastdiv_values * div_ne11 -) { - if (start_row >= end_row || start_col >= end_col) return; - const uint32_t nb1 = dst->nb[1]; // row stride in bytes - - const uint32_t ne11 = dst->ne[1]; - const uint32_t ne12 = dst->ne[2]; - const uint32_t ne11_12 = ne11 * ne12; - - const bool is_broadcast1 = (src2->ne[1] == 1); - const bool is_broadcast2 = (src2->ne[2] == 1); - const bool is_broadcast3 = (src2->ne[3] == 1); - - for (uint32_t r = start_row; r < end_row; r++) { - float * dst_row = (float *) ((uint8_t *) dst->data + r * nb1); - - uint32_t i13 = fastdiv(r, div_ne11_12); - uint32_t i12 = fastdiv(r - i13 * ne11_12, div_ne11); - uint32_t i11 = r - i13 * ne11_12 - i12 * ne11; - - uint32_t i23 = is_broadcast3 ? 0 : i13; - uint32_t i22 = is_broadcast2 ? 0 : i12; - uint32_t i21 = is_broadcast1 ? 0 : i11; - - const float * src2_row = (const float *) ((const uint8_t *) src2->data + - i21 * src2->nb[1] + i22 * src2->nb[2] + i23 * src2->nb[3]); - - float * dst_ptr = &dst_row[start_col]; - const float * src2_ptr = &src2_row[start_col]; - int remaining = end_col - start_col; - while (remaining >= 32) { - HVX_Vector v_out = hvx_vmemu(dst_ptr); - HVX_Vector v_z = hvx_vmemu(src2_ptr); - hvx_vmemu(dst_ptr) = hvx_vec_add_f32_f32(v_out, v_z); - dst_ptr += 32; - src2_ptr += 32; - remaining -= 32; - } - if (remaining > 0) { - HVX_Vector v_out = hvx_vmemu(dst_ptr); - HVX_Vector v_z = hvx_vmemu(src2_ptr); - hvx_vec_store_u(dst_ptr, remaining * sizeof(float), hvx_vec_add_f32_f32(v_out, v_z)); - } - } -} - diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-float.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-float.h new file mode 100644 index 000000000000..605892aa777c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-float.h @@ -0,0 +1,382 @@ +#ifndef HVX_MM_KERNELS_FLOAT_H +#define HVX_MM_KERNELS_FLOAT_H + +#include "hvx-utils.h" +#include "htp-tensor.h" + +// Float activation copy/quantization kernels (DDR -> VTCM) + +static inline void quantize_f32_f32_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_stride, + size_t dst_stride +) { + (void) tmp_data; + const size_t src_row_size = ne0 * sizeof(float); + for (uint32_t i = 0; i < nrows; ++i) { + hex_l2fetch(src_data, src_row_size, src_stride, 2); + hvx_copy_f32_au(dst_data, src_data, ne0); + + dst_data += dst_stride; + src_data += src_stride; + } +} + +static inline void quantize_f32_f16_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_stride, + size_t dst_stride +) { + (void) tmp_data; + const size_t src_row_size = ne0 * sizeof(float); + for (uint32_t i = 0; i < nrows; ++i) { + hex_l2fetch(src_data, src_row_size, src_stride, 2); + hvx_copy_f16_f32_au(dst_data, src_data, ne0); + + dst_data += dst_stride; + src_data += src_stride; + } +} + +static inline void quantize_f16_f16_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_stride, + size_t dst_stride +) { + (void) tmp_data; + const size_t src_row_size = ne0 * sizeof(float); + for (uint32_t i = 0; i < nrows; ++i) { + hex_l2fetch(src_data, src_row_size, src_stride, 2); + hvx_copy_f16_au(dst_data, src_data, ne0); + + dst_data += dst_stride; + src_data += src_stride; + } +} + +// Float dot product kernels (HVX) + +#if __HVX_ARCH__ < 79 +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b)) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b)) +#else +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b) +#endif + +static inline void vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_Vector * restrict x = (const HVX_Vector *) vx; + const HVX_Vector * restrict y = (const HVX_Vector *) vy; + + uint32_t nvec = n / VLEN_FP32; // num full fp32 hvx vectors + uint32_t nloe = n % VLEN_FP32; // leftover elements + + HVX_Vector rsum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + HVX_Vector prod = HVX_OP_MUL_F32(x[i], y[i]); + rsum = HVX_OP_ADD_F32(rsum, prod); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector x_sf = Q6_V_vand_QV(bmask, x[i]); + HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector prod = HVX_OP_MUL_F32(x_sf, y_sf); + rsum = HVX_OP_ADD_F32(rsum, prod); + } + + *s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum)); +} + +static inline void vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y = (const HVX_Vector *) vy0; + + uint32_t nvec = n / VLEN_FP32; + uint32_t nloe = n % VLEN_FP32; + + HVX_Vector rsum0 = Q6_V_vzero(); + HVX_Vector rsum1 = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector y_sf = y[i]; + HVX_Vector prod0 = HVX_OP_MUL_F32(x0[i], y_sf); + HVX_Vector prod1 = HVX_OP_MUL_F32(x1[i], y_sf); + rsum0 = HVX_OP_ADD_F32(rsum0, prod0); + rsum1 = HVX_OP_ADD_F32(rsum1, prod1); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector x0_sf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector x1_sf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector prod0 = HVX_OP_MUL_F32(x0_sf, y_sf); + HVX_Vector prod1 = HVX_OP_MUL_F32(x1_sf, y_sf); + rsum0 = HVX_OP_ADD_F32(rsum0, prod0); + rsum1 = HVX_OP_ADD_F32(rsum1, prod1); + } + + HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); + hvx_vec_store_u(s0, 8, rsum); +} + +static inline void vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0, const void * restrict vy1) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; + + uint32_t nvec = n / VLEN_FP32; + uint32_t nloe = n % VLEN_FP32; + + HVX_Vector r0_c0_sum = Q6_V_vzero(); + HVX_Vector r0_c1_sum = Q6_V_vzero(); + HVX_Vector r1_c0_sum = Q6_V_vzero(); + HVX_Vector r1_c1_sum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector r0_sf = x0[i]; + HVX_Vector r1_sf = x1[i]; + HVX_Vector c0_sf = y0[i]; + HVX_Vector c1_sf = y1[i]; + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + + HVX_Vector r0_sf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector r1_sf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector c0_sf = Q6_V_vand_QV(bmask, y0[i]); + HVX_Vector c1_sf = Q6_V_vand_QV(bmask, y1[i]); + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); + } + + // Reduce and store results + HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); + HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + + hvx_vec_store_u(s0, 8, r0_r1_c0_sum); + hvx_vec_store_u(s1, 8, r0_r1_c1_sum); +} + +#undef HVX_OP_ADD_F32 +#undef HVX_OP_MUL_F32 + +static inline void vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_Vector * restrict x = (const HVX_Vector *) vx; + const HVX_Vector * restrict y = (const HVX_Vector *) vy; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_VectorPair rsum_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x[i], y[i]); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf); + } + + HVX_Vector rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p))); + hvx_vec_store_u(s, 4, hvx_vec_reduce_sum_f32(rsum)); +} + +static inline void vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y = (const HVX_Vector *) vy0; + + uint32_t nvec = n / VLEN_FP16; + uint32_t nloe = n % VLEN_FP16; + + HVX_VectorPair rsum0_p = Q6_W_vzero(); + HVX_VectorPair rsum1_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector y_hf = y[i]; + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0[i], y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1[i], y_hf); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector x0_hf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector x1_hf = Q6_V_vand_QV(bmask, x1[i]); + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); + } + + HVX_Vector rsum0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p))); + HVX_Vector rsum1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p))); + HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); + hvx_vec_store_u(s0, 8, rsum); +} + +static inline void vec_dot_f16_f16_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0, const void * restrict vy1) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; + + uint32_t nvec = n / VLEN_FP16; + uint32_t nloe = n % VLEN_FP16; + + // Row sums (sf) - 4 accumulators for 2x2 tile + HVX_VectorPair r0_c0_sum_p = Q6_W_vzero(); + HVX_VectorPair r0_c1_sum_p = Q6_W_vzero(); + HVX_VectorPair r1_c0_sum_p = Q6_W_vzero(); + HVX_VectorPair r1_c1_sum_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector r0_hf = x0[i]; + HVX_Vector r1_hf = x1[i]; + HVX_Vector c0_hf = y0[i]; + HVX_Vector c1_hf = y1[i]; + + // Compute 4 dot products: r0xc0, r0xc1, r1xc0, r1xc1 + r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); + r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); + r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); + r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + + HVX_Vector r0_hf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector r1_hf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector c0_hf = Q6_V_vand_QV(bmask, y0[i]); + HVX_Vector c1_hf = Q6_V_vand_QV(bmask, y1[i]); + + r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); + r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); + r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); + r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); + } + + HVX_Vector r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c0_sum_p), Q6_V_hi_W(r0_c0_sum_p))); + HVX_Vector r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c1_sum_p), Q6_V_hi_W(r0_c1_sum_p))); + HVX_Vector r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c0_sum_p), Q6_V_hi_W(r1_c0_sum_p))); + HVX_Vector r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c1_sum_p), Q6_V_hi_W(r1_c1_sum_p))); + + // Reduce and store results + HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); + HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + + hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0 + hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1 +} + + + +static inline void hvx_tensor_add_f32_grid( + const struct htp_tensor * restrict dst, + const struct htp_tensor * restrict src2, + uint32_t start_row, + uint32_t end_row, + uint32_t start_col, + uint32_t end_col, + const struct fastdiv_values * div_ne11_12, + const struct fastdiv_values * div_ne11 +) { + if (start_row >= end_row || start_col >= end_col) return; + const uint32_t nb1 = dst->nb[1]; // row stride in bytes + + const uint32_t ne11 = dst->ne[1]; + const uint32_t ne12 = dst->ne[2]; + const uint32_t ne11_12 = ne11 * ne12; + + const bool is_broadcast1 = (src2->ne[1] == 1); + const bool is_broadcast2 = (src2->ne[2] == 1); + const bool is_broadcast3 = (src2->ne[3] == 1); + + for (uint32_t r = start_row; r < end_row; r++) { + float * dst_row = (float *) ((uint8_t *) dst->data + (size_t) r * nb1); + + uint32_t i13 = fastdiv(r, div_ne11_12); + uint32_t i12 = fastdiv(r - i13 * ne11_12, div_ne11); + uint32_t i11 = r - i13 * ne11_12 - i12 * ne11; + + uint32_t i23 = is_broadcast3 ? 0 : i13; + uint32_t i22 = is_broadcast2 ? 0 : i12; + uint32_t i21 = is_broadcast1 ? 0 : i11; + + const float * src2_row = (const float *) ((const uint8_t *) src2->data + + (size_t) i21 * src2->nb[1] + (size_t) i22 * src2->nb[2] + (size_t) i23 * src2->nb[3]); + + float * dst_ptr = &dst_row[start_col]; + const float * src2_ptr = &src2_row[start_col]; + int remaining = end_col - start_col; + while (remaining >= 32) { + HVX_Vector v_out = hvx_vmemu(dst_ptr); + HVX_Vector v_z = hvx_vmemu(src2_ptr); + hvx_vmemu(dst_ptr) = hvx_vec_add_f32_f32(v_out, v_z); + dst_ptr += 32; + src2_ptr += 32; + remaining -= 32; + } + if (remaining > 0) { + HVX_Vector v_out = hvx_vmemu(dst_ptr); + HVX_Vector v_z = hvx_vmemu(src2_ptr); + hvx_vec_store_u(dst_ptr, remaining * sizeof(float), hvx_vec_add_f32_f32(v_out, v_z)); + } + } +} + +#endif // HVX_MM_KERNELS_FLOAT_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h index 40b65aa3b550..4d6110ffafd0 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h @@ -48,22 +48,33 @@ static inline void quantize_block_f32_q8_1_tiled(float * restrict x, uint8_t * r v_sums = Q6_Vw_vadd_VwVw(v_sums, Q6_V_vror_VR(v_sums, 8)); v_sums = Q6_Vw_vadd_VwVw(v_sums, Q6_V_vror_VR(v_sums, 16)); - float vmax0[32] __attribute__((aligned(128))); - float vmax1[32] __attribute__((aligned(128))); - float vmax2[32] __attribute__((aligned(128))); - float vmax3[32] __attribute__((aligned(128))); - int32_t sums[32] __attribute__((aligned(128))); - - hvx_vec_store_u(vmax0, 128, vmax0_sf); - hvx_vec_store_u(vmax1, 128, vmax1_sf); - hvx_vec_store_u(vmax2, 128, vmax2_sf); - hvx_vec_store_u(vmax3, 128, vmax3_sf); - hvx_vec_store_u(sums, 128, v_sums); - - float d0 = vmax0[0] / 127.0f; - float d1 = vmax1[0] / 127.0f; - float d2 = vmax2[0] / 127.0f; - float d3 = vmax3[0] / 127.0f; + const HVX_Vector v_inv127 = hvx_vec_splat_f32(1.0f / 127.0f); + HVX_Vector vd0_sf = hvx_vec_mul_f32_f32(vmax0_sf, v_inv127); + HVX_Vector vd1_sf = hvx_vec_mul_f32_f32(vmax1_sf, v_inv127); + HVX_Vector vd2_sf = hvx_vec_mul_f32_f32(vmax2_sf, v_inv127); + HVX_Vector vd3_sf = hvx_vec_mul_f32_f32(vmax3_sf, v_inv127); + + HVX_Vector v_sums_sf = Q6_Vsf_equals_Vw(v_sums); + HVX_Vector voff0_sf = hvx_vec_mul_f32_f32(vd0_sf, v_sums_sf); + HVX_Vector voff1_sf = hvx_vec_mul_f32_f32(vd1_sf, Q6_V_vror_VR(v_sums_sf, 32)); + HVX_Vector voff2_sf = hvx_vec_mul_f32_f32(vd2_sf, Q6_V_vror_VR(v_sums_sf, 64)); + HVX_Vector voff3_sf = hvx_vec_mul_f32_f32(vd3_sf, Q6_V_vror_VR(v_sums_sf, 96)); + + HVX_Vector voff01_hf = hvx_vec_f32_to_f16(voff0_sf, voff1_sf); + HVX_Vector voff23_hf = hvx_vec_f32_to_f16(voff2_sf, voff3_sf); + + HVX_Vector r_scale[4] = { + hvx_vec_repl_f16(vd01_hf), + hvx_vec_repl_f16(Q6_V_vror_VR(vd01_hf, 64)), + hvx_vec_repl_f16(vd23_hf), + hvx_vec_repl_f16(Q6_V_vror_VR(vd23_hf, 64)), + }; + HVX_Vector r_offset[4] = { + hvx_vec_repl_f16(voff01_hf), + hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 64)), + hvx_vec_repl_f16(voff23_hf), + hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 64)), + }; static const uint8_t __attribute__((aligned(128))) repl[128] = { 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, @@ -89,24 +100,6 @@ static inline void quantize_block_f32_q8_1_tiled(float * restrict x, uint8_t * r HVX_Vector r6 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 24), v_repl_ctrl); HVX_Vector r7 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 28), v_repl_ctrl); - __fp16 scale_h, offset_h; - if (b == 0) { - scale_h = (__fp16) d0; - offset_h = (__fp16) (sums[0] * d0); - } else if (b == 1) { - scale_h = (__fp16) d1; - offset_h = (__fp16) (sums[8] * d1); - } else if (b == 2) { - scale_h = (__fp16) d2; - offset_h = (__fp16) (sums[16] * d2); - } else { - scale_h = (__fp16) d3; - offset_h = (__fp16) (sums[24] * d3); - } - - HVX_Vector r_scale = Q6_Vh_vsplat_R(*(int16_t *)&scale_h); - HVX_Vector r_offset = Q6_Vh_vsplat_R(*(int16_t *)&offset_h); - HVX_Vector * restrict dst = (HVX_Vector *) (y_block + b * 1280); dst[0] = r0; dst[1] = r1; @@ -116,8 +109,8 @@ static inline void quantize_block_f32_q8_1_tiled(float * restrict x, uint8_t * r dst[5] = r5; dst[6] = r6; dst[7] = r7; - dst[8] = r_scale; - dst[9] = r_offset; + dst[8] = r_scale[b]; + dst[9] = r_offset[b]; } } @@ -378,6 +371,74 @@ static inline HVX_VectorPair accum_q8_0_32x2( return Q6_W_vcombine_VV(v_sum1, v_sum0); } +// Q6_K weights are stored unsigned (0..63), see HTP_MM_WEIGHT_TILE_SIZE_Q6_K. Unpack k-group g of a tile to signed bytes (q - 32) +static inline HVX_Vector unpack_q6_k_group(const HVX_Vector * restrict vptr, int g, HVX_Vector mask_0f, HVX_Vector mask_03, HVX_Vector i32) { + HVX_Vector v_lo = (g & 1) ? Q6_Vub_vlsr_VubR(vptr[g >> 1], 4) : Q6_V_vand_VV(vptr[g >> 1], mask_0f); + HVX_Vector v_hi = (g & 3) ? Q6_Vub_vlsr_VubR(vptr[4 + (g >> 2)], 2 * (g & 3)) : vptr[4 + (g >> 2)]; + HVX_Vector v_q = Q6_V_vor_VV(v_lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(v_hi, mask_03), 4)); + return Q6_Vb_vsub_VbVb(v_q, i32); +} + +// k 0..15 and k 16..31 of a Q6_K tile have different scales: lo half of the pair sums k 0..15, hi half sums k 16..31 +static inline HVX_VectorPair accum_q6_k_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act, + HVX_Vector i32 +) { + HVX_Vector v_sum_lo = Q6_V_vzero(); + HVX_Vector v_sum_hi = Q6_V_vzero(); + HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32); + HVX_Vector v_W_hi = unpack_q6_k_group(vptr, g + 4, mask_0f, mask_03, i32); + v_sum_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum_lo, v_W_lo, v_act[g]); + v_sum_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum_hi, v_W_hi, v_act[g + 4]); + } + + return Q6_W_vcombine_VV(v_sum_hi, v_sum_lo); +} + +static inline void accum_q6_k_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_Vector i32, + HVX_VectorPair * v_sums0, + HVX_VectorPair * v_sums1 +) { + HVX_Vector v_sum0_lo = Q6_V_vzero(); + HVX_Vector v_sum0_hi = Q6_V_vzero(); + HVX_Vector v_sum1_lo = Q6_V_vzero(); + HVX_Vector v_sum1_hi = Q6_V_vzero(); + HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32); + HVX_Vector v_W_hi = unpack_q6_k_group(vptr, g + 4, mask_0f, mask_03, i32); + v_sum0_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_lo, v_W_lo, v_act0[g]); + v_sum0_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_hi, v_W_hi, v_act0[g + 4]); + v_sum1_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_lo, v_W_lo, v_act1[g]); + v_sum1_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_hi, v_W_hi, v_act1[g + 4]); + } + + *v_sums0 = Q6_W_vcombine_VV(v_sum0_hi, v_sum0_lo); + *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo); +} + +// scale the two half sums with the per-row tile scales (v_scale_w = vptr[6]) and the activation scale +static inline HVX_Vector scale_q6_k_32x1(HVX_VectorPair v_sums, HVX_Vector v_scale_w, HVX_Vector v_scale_a) { + HVX_Vector v_scale_lo = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a); + HVX_Vector v_scale_hi = hvx_vec_mul_f16_f16_to_f32_lower32(Q6_V_vror_VR(v_scale_w, 64), v_scale_a); + HVX_Vector v_lo = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(v_sums)), v_scale_lo); + HVX_Vector v_hi = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(v_sums)), v_scale_hi); + return hvx_vec_add_f32_f32(v_lo, v_hi); +} + static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -418,51 +479,7 @@ static void tiled_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float HVX_Vector i8 = Q6_Vb_vsplat_R(8); uint32_t n_k_tiles = n / 32; - uint32_t kt = 0; - for (; kt + 1 < n_k_tiles; kt += 2) { - const HVX_Vector * restrict vptr0 = (const HVX_Vector *) (tile_ptr + (kt + 0) * 640); - const HVX_Vector * restrict v_act0_0 = (const HVX_Vector *) (y0_q + (kt + 0) * 1152); - const HVX_Vector * restrict v_act1_0 = (const HVX_Vector *) (y1_q + (kt + 0) * 1152); - - const HVX_Vector * restrict vptr1 = (const HVX_Vector *) (tile_ptr + (kt + 1) * 640); - const HVX_Vector * restrict v_act0_1 = (const HVX_Vector *) (y0_q + (kt + 1) * 1152); - const HVX_Vector * restrict v_act1_1 = (const HVX_Vector *) (y1_q + (kt + 1) * 1152); - - HVX_VectorPair v_sums0 = accum_4bit_32x2(vptr0, v_act0_0, v_act1_0, i8); - HVX_VectorPair v_sums1 = accum_4bit_32x2(vptr1, v_act0_1, v_act1_1, i8); - - HVX_Vector v_sum_c0_0 = Q6_V_lo_W(v_sums0); - HVX_Vector v_sum_c1_0 = Q6_V_hi_W(v_sums0); - HVX_Vector v_sum_c0_1 = Q6_V_lo_W(v_sums1); - HVX_Vector v_sum_c1_1 = Q6_V_hi_W(v_sums1); - - HVX_Vector v_sum_sf_c0_0 = Q6_Vsf_equals_Vw(v_sum_c0_0); - HVX_Vector v_sum_sf_c1_0 = Q6_Vsf_equals_Vw(v_sum_c1_0); - HVX_Vector v_sum_sf_c0_1 = Q6_Vsf_equals_Vw(v_sum_c0_1); - HVX_Vector v_sum_sf_c1_1 = Q6_Vsf_equals_Vw(v_sum_c1_1); - - HVX_Vector v_scale_w0 = vptr0[4]; - HVX_Vector v_scale_w1 = vptr1[4]; - HVX_Vector v_scale_a_c0_0 = v_act0_0[8]; - HVX_Vector v_scale_a_c1_0 = v_act1_0[8]; - HVX_Vector v_scale_a_c0_1 = v_act0_1[8]; - HVX_Vector v_scale_a_c1_1 = v_act1_1[8]; - - HVX_Vector v_scale_comb_c0_0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w0, v_scale_a_c0_0); - HVX_Vector v_scale_comb_c1_0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w0, v_scale_a_c1_0); - HVX_Vector v_scale_comb_c0_1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w1, v_scale_a_c0_1); - HVX_Vector v_scale_comb_c1_1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w1, v_scale_a_c1_1); - - HVX_Vector v_sum_scaled_c0_0 = hvx_vec_mul_f32_f32(v_sum_sf_c0_0, v_scale_comb_c0_0); - HVX_Vector v_sum_scaled_c1_0 = hvx_vec_mul_f32_f32(v_sum_sf_c1_0, v_scale_comb_c1_0); - HVX_Vector v_sum_scaled_c0_1 = hvx_vec_mul_f32_f32(v_sum_sf_c0_1, v_scale_comb_c0_1); - HVX_Vector v_sum_scaled_c1_1 = hvx_vec_mul_f32_f32(v_sum_sf_c1_1, v_scale_comb_c1_1); - - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_add_f32_f32(v_sum_scaled_c0_0, v_sum_scaled_c0_1)); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_add_f32_f32(v_sum_scaled_c1_0, v_sum_scaled_c1_1)); - } - - for (; kt < n_k_tiles; kt++) { + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); @@ -547,76 +564,7 @@ static void tiled_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float HVX_Vector v_sum_float_c1 = Q6_V_vzero(); uint32_t n_k_tiles = n / 32; - uint32_t kt = 0; - for (; kt + 1 < n_k_tiles; kt += 2) { - const HVX_Vector * restrict vptr0 = (const HVX_Vector *) (tile_ptr + (kt + 0) * 640); - const HVX_Vector * restrict v_act0_0 = (const HVX_Vector *) (y0_q + (kt + 0) * 1280); - const HVX_Vector * restrict v_act1_0 = (const HVX_Vector *) (y1_q + (kt + 0) * 1280); - - const HVX_Vector * restrict vptr1 = (const HVX_Vector *) (tile_ptr + (kt + 1) * 640); - const HVX_Vector * restrict v_act0_1 = (const HVX_Vector *) (y0_q + (kt + 1) * 1280); - const HVX_Vector * restrict v_act1_1 = (const HVX_Vector *) (y1_q + (kt + 1) * 1280); - - HVX_VectorPair v_sums0 = accum_4bit_32x2(vptr0, v_act0_0, v_act1_0, Q6_V_vzero()); - HVX_VectorPair v_sums1 = accum_4bit_32x2(vptr1, v_act0_1, v_act1_1, Q6_V_vzero()); - - HVX_Vector v_sum_c0_0 = Q6_V_lo_W(v_sums0); - HVX_Vector v_sum_c1_0 = Q6_V_hi_W(v_sums0); - HVX_Vector v_sum_c0_1 = Q6_V_lo_W(v_sums1); - HVX_Vector v_sum_c1_1 = Q6_V_hi_W(v_sums1); - - HVX_Vector v_sum_sf_c0_0 = Q6_Vsf_equals_Vw(v_sum_c0_0); - HVX_Vector v_sum_sf_c1_0 = Q6_Vsf_equals_Vw(v_sum_c1_0); - HVX_Vector v_sum_sf_c0_1 = Q6_Vsf_equals_Vw(v_sum_c0_1); - HVX_Vector v_sum_sf_c1_1 = Q6_Vsf_equals_Vw(v_sum_c1_1); - - HVX_Vector v_scale_offset0 = vptr0[4]; - HVX_VectorPair p_deal0 = Q6_W_vdeal_VVR(v_scale_offset0, v_scale_offset0, -2); - HVX_Vector v_scale0 = Q6_V_lo_W(p_deal0); - HVX_Vector v_offset0 = Q6_V_hi_W(p_deal0); - - HVX_Vector v_scale_offset1 = vptr1[4]; - HVX_VectorPair p_deal1 = Q6_W_vdeal_VVR(v_scale_offset1, v_scale_offset1, -2); - HVX_Vector v_scale1 = Q6_V_lo_W(p_deal1); - HVX_Vector v_offset1 = Q6_V_hi_W(p_deal1); - - HVX_Vector v_scale_a_c0_0 = v_act0_0[8]; - HVX_Vector v_sum_a_c0_0 = v_act0_0[9]; - HVX_Vector v_scale_a_c1_0 = v_act1_0[8]; - HVX_Vector v_sum_a_c1_0 = v_act1_0[9]; - - HVX_Vector v_scale_a_c0_1 = v_act0_1[8]; - HVX_Vector v_sum_a_c0_1 = v_act0_1[9]; - HVX_Vector v_scale_a_c1_1 = v_act1_1[8]; - HVX_Vector v_sum_a_c1_1 = v_act1_1[9]; - - HVX_Vector v_scale_comb_c0_0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale0, v_scale_a_c0_0); - HVX_Vector v_offset_comb_c0_0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset0, v_sum_a_c0_0); - HVX_Vector v_scale_comb_c1_0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale0, v_scale_a_c1_0); - HVX_Vector v_offset_comb_c1_0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset0, v_sum_a_c1_0); - - HVX_Vector v_scale_comb_c0_1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale1, v_scale_a_c0_1); - HVX_Vector v_offset_comb_c0_1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset1, v_sum_a_c0_1); - HVX_Vector v_scale_comb_c1_1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale1, v_scale_a_c1_1); - HVX_Vector v_offset_comb_c1_1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset1, v_sum_a_c1_1); - - HVX_Vector v_scaled_dot_c0_0 = hvx_vec_mul_f32_f32(v_sum_sf_c0_0, v_scale_comb_c0_0); - HVX_Vector v_sum_scaled_c0_0 = hvx_vec_add_f32_f32(v_scaled_dot_c0_0, v_offset_comb_c0_0); - - HVX_Vector v_scaled_dot_c1_0 = hvx_vec_mul_f32_f32(v_sum_sf_c1_0, v_scale_comb_c1_0); - HVX_Vector v_sum_scaled_c1_0 = hvx_vec_add_f32_f32(v_scaled_dot_c1_0, v_offset_comb_c1_0); - - HVX_Vector v_scaled_dot_c0_1 = hvx_vec_mul_f32_f32(v_sum_sf_c0_1, v_scale_comb_c0_1); - HVX_Vector v_sum_scaled_c0_1 = hvx_vec_add_f32_f32(v_scaled_dot_c0_1, v_offset_comb_c0_1); - - HVX_Vector v_scaled_dot_c1_1 = hvx_vec_mul_f32_f32(v_sum_sf_c1_1, v_scale_comb_c1_1); - HVX_Vector v_sum_scaled_c1_1 = hvx_vec_add_f32_f32(v_scaled_dot_c1_1, v_offset_comb_c1_1); - - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_add_f32_f32(v_sum_scaled_c0_0, v_sum_scaled_c0_1)); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_add_f32_f32(v_sum_scaled_c1_0, v_sum_scaled_c1_1)); - } - - for (; kt < n_k_tiles; kt++) { + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280); const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280); @@ -703,51 +651,7 @@ static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float HVX_Vector v_sum_float_c1 = Q6_V_vzero(); uint32_t n_k_tiles = n / 32; - uint32_t kt = 0; - for (; kt + 1 < n_k_tiles; kt += 2) { - const HVX_Vector * restrict vptr0 = (const HVX_Vector *) (tile_ptr + (kt + 0) * 1152); - const HVX_Vector * restrict v_act0_0 = (const HVX_Vector *) (y0_q + (kt + 0) * 1152); - const HVX_Vector * restrict v_act1_0 = (const HVX_Vector *) (y1_q + (kt + 0) * 1152); - - const HVX_Vector * restrict vptr1 = (const HVX_Vector *) (tile_ptr + (kt + 1) * 1152); - const HVX_Vector * restrict v_act0_1 = (const HVX_Vector *) (y0_q + (kt + 1) * 1152); - const HVX_Vector * restrict v_act1_1 = (const HVX_Vector *) (y1_q + (kt + 1) * 1152); - - HVX_VectorPair v_sums0 = accum_q8_0_32x2(vptr0, v_act0_0, v_act1_0); - HVX_VectorPair v_sums1 = accum_q8_0_32x2(vptr1, v_act0_1, v_act1_1); - - HVX_Vector v_sum_c0_0 = Q6_V_lo_W(v_sums0); - HVX_Vector v_sum_c1_0 = Q6_V_hi_W(v_sums0); - HVX_Vector v_sum_c0_1 = Q6_V_lo_W(v_sums1); - HVX_Vector v_sum_c1_1 = Q6_V_hi_W(v_sums1); - - HVX_Vector v_sum_sf_c0_0 = Q6_Vsf_equals_Vw(v_sum_c0_0); - HVX_Vector v_sum_sf_c1_0 = Q6_Vsf_equals_Vw(v_sum_c1_0); - HVX_Vector v_sum_sf_c0_1 = Q6_Vsf_equals_Vw(v_sum_c0_1); - HVX_Vector v_sum_sf_c1_1 = Q6_Vsf_equals_Vw(v_sum_c1_1); - - HVX_Vector v_scale_w0 = vptr0[8]; - HVX_Vector v_scale_w1 = vptr1[8]; - HVX_Vector v_scale_a_c0_0 = v_act0_0[8]; - HVX_Vector v_scale_a_c1_0 = v_act1_0[8]; - HVX_Vector v_scale_a_c0_1 = v_act0_1[8]; - HVX_Vector v_scale_a_c1_1 = v_act1_1[8]; - - HVX_Vector v_scale_comb_c0_0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w0, v_scale_a_c0_0); - HVX_Vector v_scale_comb_c1_0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w0, v_scale_a_c1_0); - HVX_Vector v_scale_comb_c0_1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w1, v_scale_a_c0_1); - HVX_Vector v_scale_comb_c1_1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w1, v_scale_a_c1_1); - - HVX_Vector v_sum_scaled_c0_0 = hvx_vec_mul_f32_f32(v_sum_sf_c0_0, v_scale_comb_c0_0); - HVX_Vector v_sum_scaled_c1_0 = hvx_vec_mul_f32_f32(v_sum_sf_c1_0, v_scale_comb_c1_0); - HVX_Vector v_sum_scaled_c0_1 = hvx_vec_mul_f32_f32(v_sum_sf_c0_1, v_scale_comb_c0_1); - HVX_Vector v_sum_scaled_c1_1 = hvx_vec_mul_f32_f32(v_sum_sf_c1_1, v_scale_comb_c1_1); - - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_add_f32_f32(v_sum_scaled_c0_0, v_sum_scaled_c0_1)); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_add_f32_f32(v_sum_scaled_c1_0, v_sum_scaled_c1_1)); - } - - for (; kt < n_k_tiles; kt++) { + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 1152); const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); @@ -785,6 +689,63 @@ static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float } } +static void tiled_vec_dot_q6_k_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + HVX_Vector i32 = Q6_Vb_vsplat_R(32); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896); + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1152); + + HVX_VectorPair v_sums = accum_q6_k_32x1(vptr, v_act, i32); + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, scale_q6_k_32x1(v_sums, vptr[6], v_act[8])); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q6_k_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + HVX_Vector i32 = Q6_Vb_vsplat_R(32); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896); + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); + + HVX_VectorPair v_sums0, v_sums1; + accum_q6_k_32x2(vptr, v_act0, v_act1, i32, &v_sums0, &v_sums1); + + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, scale_q6_k_32x1(v_sums0, vptr[6], v_act0[8])); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, scale_q6_k_32x1(v_sums1, vptr[6], v_act1[8])); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -827,51 +788,7 @@ static void tiled_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, floa HVX_Vector lut = *(const HVX_Vector *) kvalues_iq4nl_lut; uint32_t n_k_tiles = n / 32; - uint32_t kt = 0; - for (; kt + 1 < n_k_tiles; kt += 2) { - const HVX_Vector * restrict vptr0 = (const HVX_Vector *) (tile_ptr + (kt + 0) * 640); - const HVX_Vector * restrict v_act0_0 = (const HVX_Vector *) (y0_q + (kt + 0) * 1152); - const HVX_Vector * restrict v_act1_0 = (const HVX_Vector *) (y1_q + (kt + 0) * 1152); - - const HVX_Vector * restrict vptr1 = (const HVX_Vector *) (tile_ptr + (kt + 1) * 640); - const HVX_Vector * restrict v_act0_1 = (const HVX_Vector *) (y0_q + (kt + 1) * 1152); - const HVX_Vector * restrict v_act1_1 = (const HVX_Vector *) (y1_q + (kt + 1) * 1152); - - HVX_VectorPair v_sums0 = accum_4bit_32x2_lut(vptr0, v_act0_0, v_act1_0, mask_h4, lut); - HVX_VectorPair v_sums1 = accum_4bit_32x2_lut(vptr1, v_act0_1, v_act1_1, mask_h4, lut); - - HVX_Vector v_sum_c0_0 = Q6_V_lo_W(v_sums0); - HVX_Vector v_sum_c1_0 = Q6_V_hi_W(v_sums0); - HVX_Vector v_sum_c0_1 = Q6_V_lo_W(v_sums1); - HVX_Vector v_sum_c1_1 = Q6_V_hi_W(v_sums1); - - HVX_Vector v_sum_sf_c0_0 = Q6_Vsf_equals_Vw(v_sum_c0_0); - HVX_Vector v_sum_sf_c1_0 = Q6_Vsf_equals_Vw(v_sum_c1_0); - HVX_Vector v_sum_sf_c0_1 = Q6_Vsf_equals_Vw(v_sum_c0_1); - HVX_Vector v_sum_sf_c1_1 = Q6_Vsf_equals_Vw(v_sum_c1_1); - - HVX_Vector v_scale_w0 = vptr0[4]; - HVX_Vector v_scale_w1 = vptr1[4]; - HVX_Vector v_scale_a_c0_0 = v_act0_0[8]; - HVX_Vector v_scale_a_c1_0 = v_act1_0[8]; - HVX_Vector v_scale_a_c0_1 = v_act0_1[8]; - HVX_Vector v_scale_a_c1_1 = v_act1_1[8]; - - HVX_Vector v_scale_comb_c0_0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w0, v_scale_a_c0_0); - HVX_Vector v_scale_comb_c1_0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w0, v_scale_a_c1_0); - HVX_Vector v_scale_comb_c0_1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w1, v_scale_a_c0_1); - HVX_Vector v_scale_comb_c1_1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w1, v_scale_a_c1_1); - - HVX_Vector v_sum_scaled_c0_0 = hvx_vec_mul_f32_f32(v_sum_sf_c0_0, v_scale_comb_c0_0); - HVX_Vector v_sum_scaled_c1_0 = hvx_vec_mul_f32_f32(v_sum_sf_c1_0, v_scale_comb_c1_0); - HVX_Vector v_sum_scaled_c0_1 = hvx_vec_mul_f32_f32(v_sum_sf_c0_1, v_scale_comb_c0_1); - HVX_Vector v_sum_scaled_c1_1 = hvx_vec_mul_f32_f32(v_sum_sf_c1_1, v_scale_comb_c1_1); - - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_add_f32_f32(v_sum_scaled_c0_0, v_sum_scaled_c0_1)); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_add_f32_f32(v_sum_scaled_c1_0, v_sum_scaled_c1_1)); - } - - for (; kt < n_k_tiles; kt++) { + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); @@ -964,69 +881,7 @@ static void tiled_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, floa HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff); uint32_t n_k_tiles = n / 32; - uint32_t kt = 0; - for (; kt + 1 < n_k_tiles; kt += 2) { - const HVX_Vector * restrict vptr0 = (const HVX_Vector *) (tile_ptr + (kt + 0) * 640); - const HVX_Vector * restrict v_act0_0 = (const HVX_Vector *) (y0_q + (kt + 0) * 1152); - const HVX_Vector * restrict v_act1_0 = (const HVX_Vector *) (y1_q + (kt + 0) * 1152); - - const HVX_Vector * restrict vptr1 = (const HVX_Vector *) (tile_ptr + (kt + 1) * 640); - const HVX_Vector * restrict v_act0_1 = (const HVX_Vector *) (y0_q + (kt + 1) * 1152); - const HVX_Vector * restrict v_act1_1 = (const HVX_Vector *) (y1_q + (kt + 1) * 1152); - - HVX_VectorPair v_sums0 = accum_4bit_32x2_lut(vptr0, v_act0_0, v_act1_0, mask_h4, lut); - HVX_VectorPair v_sums1 = accum_4bit_32x2_lut(vptr1, v_act0_1, v_act1_1, mask_h4, lut); - - HVX_Vector v_sum_c0_0 = Q6_V_lo_W(v_sums0); - HVX_Vector v_sum_c1_0 = Q6_V_hi_W(v_sums0); - HVX_Vector v_sum_c0_1 = Q6_V_lo_W(v_sums1); - HVX_Vector v_sum_c1_1 = Q6_V_hi_W(v_sums1); - - HVX_Vector v_sum_sf_c0_0 = Q6_Vsf_equals_Vw(v_sum_c0_0); - HVX_Vector v_sum_sf_c1_0 = Q6_Vsf_equals_Vw(v_sum_c1_0); - HVX_Vector v_sum_sf_c0_1 = Q6_Vsf_equals_Vw(v_sum_c0_1); - HVX_Vector v_sum_sf_c1_1 = Q6_Vsf_equals_Vw(v_sum_c1_1); - - HVX_Vector v_scale_w0 = hvx_vmem(tile_ptr + (kt + 0) * 640 + 512); - HVX_Vector r0_d0 = Q6_V_vdelta_VV(v_scale_w0, expand); - r0_d0 = Q6_V_vand_VV(r0_d0, e8m0_mask); - HVX_Vector v_scale_w_f32_0 = Q6_Vw_vasl_VwR(r0_d0, 23); - - HVX_Vector v_scale_w1 = hvx_vmem(tile_ptr + (kt + 1) * 640 + 512); - HVX_Vector r0_d1 = Q6_V_vdelta_VV(v_scale_w1, expand); - r0_d1 = Q6_V_vand_VV(r0_d1, e8m0_mask); - HVX_Vector v_scale_w_f32_1 = Q6_Vw_vasl_VwR(r0_d1, 23); - - HVX_Vector v_scale_a_c0_f16_0 = v_act0_0[8]; - HVX_Vector v_scale_a_c1_f16_0 = v_act1_0[8]; - HVX_Vector v_scale_a_c0_f16_1 = v_act0_1[8]; - HVX_Vector v_scale_a_c1_f16_1 = v_act1_1[8]; - - HVX_VectorPair p_scale_a_c0_f32_0 = hvx_vec_f16_to_f32_shuff(v_scale_a_c0_f16_0); - HVX_VectorPair p_scale_a_c1_f32_0 = hvx_vec_f16_to_f32_shuff(v_scale_a_c1_f16_0); - HVX_VectorPair p_scale_a_c0_f32_1 = hvx_vec_f16_to_f32_shuff(v_scale_a_c0_f16_1); - HVX_VectorPair p_scale_a_c1_f32_1 = hvx_vec_f16_to_f32_shuff(v_scale_a_c1_f16_1); - - HVX_Vector v_scale_a_c0_0 = Q6_V_lo_W(p_scale_a_c0_f32_0); - HVX_Vector v_scale_a_c1_0 = Q6_V_lo_W(p_scale_a_c1_f32_0); - HVX_Vector v_scale_a_c0_1 = Q6_V_lo_W(p_scale_a_c0_f32_1); - HVX_Vector v_scale_a_c1_1 = Q6_V_lo_W(p_scale_a_c1_f32_1); - - HVX_Vector v_scale_comb_c0_0 = hvx_vec_mul_f32_f32(v_scale_w_f32_0, v_scale_a_c0_0); - HVX_Vector v_scale_comb_c1_0 = hvx_vec_mul_f32_f32(v_scale_w_f32_0, v_scale_a_c1_0); - HVX_Vector v_scale_comb_c0_1 = hvx_vec_mul_f32_f32(v_scale_w_f32_1, v_scale_a_c0_1); - HVX_Vector v_scale_comb_c1_1 = hvx_vec_mul_f32_f32(v_scale_w_f32_1, v_scale_a_c1_1); - - HVX_Vector v_sum_scaled_c0_0 = hvx_vec_mul_f32_f32(v_sum_sf_c0_0, v_scale_comb_c0_0); - HVX_Vector v_sum_scaled_c1_0 = hvx_vec_mul_f32_f32(v_sum_sf_c1_0, v_scale_comb_c1_0); - HVX_Vector v_sum_scaled_c0_1 = hvx_vec_mul_f32_f32(v_sum_sf_c0_1, v_scale_comb_c0_1); - HVX_Vector v_sum_scaled_c1_1 = hvx_vec_mul_f32_f32(v_sum_sf_c1_1, v_scale_comb_c1_1); - - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_add_f32_f32(v_sum_scaled_c0_0, v_sum_scaled_c0_1)); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_add_f32_f32(v_sum_scaled_c1_0, v_sum_scaled_c1_1)); - } - - for (; kt < n_k_tiles; kt++) { + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); diff --git a/ggml/src/ggml-hexagon/htp/hvx-scale.h b/ggml/src/ggml-hexagon/htp/hvx-scale.h index 9b1a28f529a2..5d0650307ef8 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-scale.h +++ b/ggml/src/ggml-hexagon/htp/hvx-scale.h @@ -68,30 +68,30 @@ static inline void hvx_scale_f32(uint8_t * restrict dst, const uint8_t * restric } } -#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - HVX_Vector vs = hvx_vec_splat_f32(scale); \ - HVX_Vector vo = hvx_vec_splat_f32(offset); \ - \ - const uint32_t elem_size = sizeof(float); \ - const uint32_t epv = 128 / elem_size; \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; ++i) { \ +#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + HVX_Vector vs = hvx_vec_splat_f32(scale); \ + HVX_Vector vo = hvx_vec_splat_f32(offset); \ + \ + const uint32_t elem_size = sizeof(float); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; ++i) { \ HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \ - vdst[i] = Q6_Vsf_equals_Vqf32(v); \ - } \ - if (nloe) { \ + vdst[i] = Q6_Vsf_equals_Vqf32(v); \ + } \ + if (nloe) { \ HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \ - vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \ - } \ + vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \ + } \ } while(0) static inline void hvx_scale_offset_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h index dd66dd84c95a..552017309d19 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h @@ -68,50 +68,50 @@ static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) { return Q6_Vsf_equals_Vqf32(res); } -#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const HVX_Vector one = hvx_vec_splat_f32(1.f); \ - const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \ - const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \ - \ - const uint32_t epv = 128 / sizeof(float); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ - } \ - if (nloe) { \ +#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const HVX_Vector one = hvx_vec_splat_f32(1.f); \ + const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \ + const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \ + \ + const uint32_t epv = 128 / sizeof(float); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ + } \ + if (nloe) { \ HVX_Vector tmp = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ - vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ - } \ + vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ + } \ } while(0) -#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const uint32_t epv = 128 / sizeof(float); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \ - } \ - if (nloe) { \ - HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \ +#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / sizeof(float); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \ vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ - } \ + } \ } while(0) static inline void hvx_sigmoid_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { diff --git a/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h index c5b9a5d47c17..8648af0e5b95 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h @@ -4,87 +4,75 @@ #include "hvx-base.h" #include "hvx-floor.h" -static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) { - HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f); - HVX_Vector const_half = hvx_vec_splat_f32(0.5f); - HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f); - HVX_Vector const_one = hvx_vec_splat_f32(1.0f); +// Range-reduce x to y in [-pi/2, pi/2] and the quadrant sign (-1)^n. +// Floor/truncate need IEEE bits, so convert qf32 back to sf before them. +static inline void hvx_vec_sincos_reduce_f32(HVX_Vector x, HVX_Vector * y, HVX_Vector * sign) { + HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f); + HVX_Vector const_half = hvx_vec_splat_f32(0.5f); + HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f); + HVX_Vector const_one = hvx_vec_splat_f32(1.0f); HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f); + HVX_Vector const_one_i = Q6_V_vsplat_R(1); + + HVX_Vector x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, const_inv_pi)); + x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(x_over_pi, const_half)); - // n = floor(x * (1/pi) + 0.5) - HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half)); + HVX_Vector n_float = hvx_vec_floor_f32(x_over_pi); + HVX_Vector n_int = hvx_vec_truncate_f32(n_float); - // y = x - n * pi - HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi)); + HVX_Vector n_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(n_float, const_pi)); + *y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(x, n_pi)); - // Sign determination: if n is odd, sign is -1.0f, else 1.0f - // half_n = n * 0.5f - HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half); - // floor_half_n = floor(half_n) - HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n); - // is_odd = half_n > floor_half_n - HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n); - // sign = vmux(is_odd, -1.0f, 1.0f) - HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one); + HVX_VectorPred is_odd = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(n_int, const_one_i), const_one_i); + *sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one); +} - // z = y^2 - HVX_Vector z = hvx_vec_mul_f32_f32(y, y); +static inline void hvx_vec_sincos_f32(HVX_Vector x, HVX_Vector * vcos, HVX_Vector * vsin) { + HVX_Vector y; + HVX_Vector sign; + hvx_vec_sincos_reduce_f32(x, &y, &sign); + + HVX_Vector z = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, y)); - // Chebyshev approximation for cos(y) HVX_Vector c4 = hvx_vec_splat_f32(2.3557242013849433e-05f); HVX_Vector c3 = hvx_vec_splat_f32(-0.0013871428263450528f); HVX_Vector c2 = hvx_vec_splat_f32(0.041665895266688284f); HVX_Vector c1 = hvx_vec_splat_f32(-0.4999999360426369f); HVX_Vector c0 = hvx_vec_splat_f32(0.9999999999071725f); - HVX_Vector cos_y = hvx_vec_add_f32_f32(c3, hvx_vec_mul_f32_f32(z, c4)); - cos_y = hvx_vec_add_f32_f32(c2, hvx_vec_mul_f32_f32(z, cos_y)); - cos_y = hvx_vec_add_f32_f32(c1, hvx_vec_mul_f32_f32(z, cos_y)); - cos_y = hvx_vec_add_f32_f32(c0, hvx_vec_mul_f32_f32(z, cos_y)); - - return hvx_vec_mul_f32_f32(cos_y, sign); -} - -static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) { - HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f); - HVX_Vector const_half = hvx_vec_splat_f32(0.5f); - HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f); - HVX_Vector const_one = hvx_vec_splat_f32(1.0f); - HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f); - - // n = floor(x * (1/pi) + 0.5) - HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half)); + HVX_Vector cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, c4)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); - // y = x - n * pi - HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi)); - - // Sign determination: if n is odd, sign is -1.0f, else 1.0f - // half_n = n * 0.5f - HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half); - // floor_half_n = floor(half_n) - HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n); - // is_odd = half_n > floor_half_n - HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n); - // sign = vmux(is_odd, -1.0f, 1.0f) - HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one); - - // z = y^2 - HVX_Vector z = hvx_vec_mul_f32_f32(y, y); - - // Chebyshev approximation for sin(y) HVX_Vector s4 = hvx_vec_splat_f32(2.642186986152672e-06f); HVX_Vector s3 = hvx_vec_splat_f32(-0.00019825318964070864f); HVX_Vector s2 = hvx_vec_splat_f32(0.00833326283319605f); HVX_Vector s1 = hvx_vec_splat_f32(-0.16666666082087775f); HVX_Vector s0 = hvx_vec_splat_f32(0.999999999915155f); - HVX_Vector sin_y = hvx_vec_add_f32_f32(s3, hvx_vec_mul_f32_f32(z, s4)); - sin_y = hvx_vec_add_f32_f32(s2, hvx_vec_mul_f32_f32(z, sin_y)); - sin_y = hvx_vec_add_f32_f32(s1, hvx_vec_mul_f32_f32(z, sin_y)); - sin_y = hvx_vec_add_f32_f32(s0, hvx_vec_mul_f32_f32(z, sin_y)); - sin_y = hvx_vec_mul_f32_f32(y, sin_y); + HVX_Vector sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, s4)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, sin_y)); + + *vcos = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(cos_y, sign)); + *vsin = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(sin_y, sign)); +} - return hvx_vec_mul_f32_f32(sin_y, sign); +static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) { + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(x, &vcos, &vsin); + return vcos; +} + +static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) { + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(x, &vcos, &vsin); + return vsin; } #endif /* HVX_SIN_COS_H */ diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.c index 35fc103df8fe..2e05cf3e1002 100644 --- a/ggml/src/ggml-hexagon/htp/im2col-ops.c +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c @@ -3,24 +3,30 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include <HAP_farf.h> -#include <HAP_perf.h> #include <hexagon_protos.h> #include <hexagon_types.h> #include <string.h> +#include "hex-common.h" + #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" #include "hvx-utils.h" -#include "hex-dma.h" +#include "dma-queue.h" #include "hex-profile.h" #include "htp-vtcm.h" +#include "htp-tensor.h" struct htp_im2col_context { struct htp_ops_context * octx; + uint32_t patch_base; // first patch index assigned to this dev + uint32_t npatches; // number of patches assigned to this dev uint32_t npatches_per_thread; // patches = N*OH*OW (pure-DDR kernel) + uint32_t pe_row_base; // first N*OH row index assigned to this dev (DMA path) + uint32_t pe_nrows; // number of N*OH rows assigned to this dev (DMA path) uint32_t pe_rows_per_thread; // N*OH rows per worker uint32_t pe_src_row_bytes; // one output row's source: IC*KH*IW*4, rounded 256 uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256 @@ -30,6 +36,9 @@ struct htp_im2col_context { uint8_t * pe_vtcm_dst; // base of the 2x dst buffers region uint32_t pe_src_size_per_thread; // 2 * pe_src_row_bytes uint32_t pe_dst_size_per_thread; // 2 * pe_dst_row_bytes + + uint32_t pe_owb; // output-col block size + uint32_t pe_wb; // staged source window width }; // Per-op VTCM layout for the patch-embed DMA path @@ -53,152 +62,324 @@ static inline void htp_im2col_vtcm_layout_build(struct htp_im2col_vtcm_layout * L->total_bytes = L->off_dst + L->dst_bytes_per_thread * n_threads; } -#define IM2COL_PATCHEMBED_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN, DST_ELEM, TAG) \ - static void FNAME(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; \ - struct htp_ops_context * octx = ictx->octx; \ - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; \ - const struct htp_tensor * restrict src1 = octx->src[1]; \ - const struct htp_tensor * restrict dst = octx->dst; \ - const int32_t s0 = octx->op_params[0]; \ - const int32_t s1 = octx->op_params[1]; \ - const int32_t p0 = octx->op_params[2]; \ - const int32_t p1 = octx->op_params[3]; \ - const int32_t d0 = octx->op_params[4]; \ - const int32_t d1 = octx->op_params[5]; \ - const uint32_t N = src1->ne[3]; \ - const uint32_t IC = src1->ne[2]; \ - const uint32_t IH = src1->ne[1]; \ - const uint32_t IW = src1->ne[0]; \ - const uint32_t KH = octx->src[0]->ne[1]; \ - const uint32_t KW = octx->src[0]->ne[0]; \ - const uint32_t OH = dst->ne[2]; \ - const uint32_t OW = dst->ne[1]; \ - const uint32_t patch_stride = IC * KH * KW; \ - const float * restrict src_data = (const float *) src1->data; \ - DST_CTYPE * restrict dst_data = (DST_CTYPE *) dst->data; \ - const uint32_t npatches = N * OH * OW; \ - const uint32_t patch_start = ictx->npatches_per_thread * ith; \ - const uint32_t patch_end = MIN(patch_start + ictx->npatches_per_thread, npatches); \ - if (patch_start >= patch_end) { \ - return; \ - } \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, patch_start); \ - for (uint32_t p = patch_start; p < patch_end; p++) { \ - const uint32_t iow = p % OW; \ - const uint32_t ioh = (p / OW) % OH; \ - const uint32_t in = p / (OW * OH); \ - DST_CTYPE * restrict dst_patch = dst_data + (uint64_t) p * patch_stride; \ - for (uint32_t iic = 0; iic < IC; iic++) { \ - const float * restrict src_plane = src_data + ((uint64_t) in * IC + iic) * IH * IW; \ - for (uint32_t ikh = 0; ikh < KH; ikh++) { \ - const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \ - DST_CTYPE * restrict out_run = dst_patch + iic * (KH * KW) + ikh * KW; \ - if (iih < 0 || iih >= (int32_t) IH) { \ - SPLAT_FN(out_run, 0.0f, KW); \ - continue; \ - } \ - const int32_t iiw0 = (int32_t) iow * s0 - p0; \ - const float * restrict src_run = src_plane + (uint64_t) iih * IW + iiw0; \ - if (d0 == 1) { \ - /* contiguous source run: [lo,hi) is in-bounds, tails are zero pad */ \ - const int32_t lo = iiw0 < 0 ? -iiw0 : 0; \ - int32_t hi = (int32_t) IW - iiw0; \ - if (hi > (int32_t) KW) { \ - hi = (int32_t) KW; \ - } \ - if (hi <= lo) { \ - SPLAT_FN(out_run, 0.0f, KW); \ - } else { \ - if (lo > 0) { \ - SPLAT_FN(out_run, 0.0f, (uint32_t) lo); \ - } \ - COPY_FN((uint8_t *) (out_run + lo), (const uint8_t *) (src_run + lo), \ - (uint32_t) (hi - lo)); \ - if (hi < (int32_t) KW) { \ - SPLAT_FN(out_run + hi, 0.0f, (KW - (uint32_t) hi)); \ - } \ - } \ - continue; \ - } \ - for (uint32_t ikw = 0; ikw < KW; ikw++) { \ - const int32_t iiw = (int32_t) iow * s0 + (int32_t) ikw * d0 - p0; \ - out_run[ikw] = (iiw < 0 || iiw >= (int32_t) IW) ? \ - (DST_CTYPE) 0.0f : \ - (DST_CTYPE) src_plane[(uint64_t) iih * IW + iiw]; \ - } \ - } \ - } \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, patch_start); \ +#define IM2COL_PATCHEMBED_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN, DST_ELEM, TAG) \ + static void FNAME(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; \ + struct htp_ops_context * octx = ictx->octx; \ + struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; \ + const struct htp_tensor * restrict src0 = octx->src[0]; \ + const struct htp_tensor * restrict src1 = octx->src[1]; \ + const struct htp_tensor * restrict dst = octx->dst; \ + const int32_t s0 = octx->op_params[0]; \ + const int32_t s1 = octx->op_params[1]; \ + const int32_t p0 = octx->op_params[2]; \ + const int32_t p1 = octx->op_params[3]; \ + const int32_t d0 = octx->op_params[4]; \ + const int32_t d1 = octx->op_params[5]; \ + const int32_t is_2D = octx->op_params[6] == 1; \ + const uint32_t N = is_2D ? src1->ne[3] : src1->ne[2]; \ + const uint32_t IC = is_2D ? src1->ne[2] : src1->ne[1]; \ + const uint32_t IH = is_2D ? src1->ne[1] : 1; \ + const uint32_t IW = src1->ne[0]; \ + const uint32_t KH = is_2D ? src0->ne[1] : 1; \ + const uint32_t KW = src0->ne[0]; \ + const uint32_t OH = is_2D ? dst->ne[2] : 1; \ + const uint32_t OW = dst->ne[1]; \ + const uint32_t patch_stride = IC * KH * KW; \ + const float * restrict src_data = (const float *) (uintptr_t) src1->data; \ + DST_CTYPE * restrict dst_data = (DST_CTYPE *) (uintptr_t) dst->data; \ + const uint32_t patch_end = ictx->patch_base + ictx->npatches; \ + const uint32_t patch_start = ictx->patch_base + ictx->npatches_per_thread * ith; \ + const uint32_t patch_stop = MIN(patch_start + ictx->npatches_per_thread, patch_end); \ + if (patch_start >= patch_stop) { \ + return; \ + } \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, patch_start); \ + for (uint32_t p = patch_start; p < patch_stop; p++) { \ + const uint32_t iow = p % OW; \ + const uint32_t ioh = (p / OW) % OH; \ + const uint32_t in = p / (OW * OH); \ + DST_CTYPE * restrict dst_patch = dst_data + (uint64_t) p * patch_stride; \ + for (uint32_t iic = 0; iic < IC; iic++) { \ + const float * restrict src_plane = src_data + ((uint64_t) in * IC + iic) * IH * IW; \ + for (uint32_t ikh = 0; ikh < KH; ikh++) { \ + const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \ + DST_CTYPE * restrict out_run = dst_patch + iic * (KH * KW) + ikh * KW; \ + if (iih < 0 || iih >= (int32_t) IH) { \ + SPLAT_FN(out_run, 0.0f, KW); \ + continue; \ + } \ + const int32_t iiw0 = (int32_t) iow * s0 - p0; \ + const float * restrict src_run = src_plane + (uint64_t) iih * IW + iiw0; \ + if (d0 == 1) { \ + /* contiguous source run: [lo,hi) is in-bounds, tails are zero pad */ \ + const int32_t lo = iiw0 < 0 ? -iiw0 : 0; \ + int32_t hi = (int32_t) IW - iiw0; \ + if (hi > (int32_t) KW) { \ + hi = (int32_t) KW; \ + } \ + if (hi <= lo) { \ + SPLAT_FN(out_run, 0.0f, KW); \ + } else { \ + if (lo > 0) { \ + SPLAT_FN(out_run, 0.0f, (uint32_t) lo); \ + } \ + COPY_FN((uint8_t *) (out_run + lo), (const uint8_t *) (src_run + lo), \ + (uint32_t) (hi - lo)); \ + if (hi < (int32_t) KW) { \ + SPLAT_FN(out_run + hi, 0.0f, (KW - (uint32_t) hi)); \ + } \ + } \ + continue; \ + } \ + for (uint32_t ikw = 0; ikw < KW; ikw++) { \ + const int32_t iiw = (int32_t) iow * s0 + (int32_t) ikw * d0 - p0; \ + out_run[ikw] = (iiw < 0 || iiw >= (int32_t) IW) ? \ + (DST_CTYPE) 0.0f : \ + (DST_CTYPE) src_plane[(uint64_t) iih * IW + iiw]; \ + } \ + } \ + } \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, patch_start); \ } IM2COL_PATCHEMBED_BODY(im2col_patchembed_thread, __fp16, hvx_copy_f16_f32_uu, hvx_splat_f16_u, sizeof(__fp16), "f32-f16") IM2COL_PATCHEMBED_BODY(im2col_patchembed_f32_thread, float, hvx_copy_f32_uu, hvx_splat_f32_u, sizeof(float), "f32-f32") -#define IM2COL_PATCHEMBED_DMA_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN, DST_ELEM, TAG) \ - static void FNAME(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; \ - struct htp_ops_context * octx = ictx->octx; \ - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; \ - const struct htp_tensor * restrict src1 = octx->src[1]; \ - const struct htp_tensor * restrict dst = octx->dst; \ - const uint32_t N = src1->ne[3], IC = src1->ne[2], IH = src1->ne[1], IW = src1->ne[0]; \ - const uint32_t KH = octx->src[0]->ne[1], KW = octx->src[0]->ne[0]; \ - const uint32_t OH = dst->ne[2], OW = dst->ne[1]; \ - const uint32_t patch_stride = IC * KH * KW; \ - const float * restrict src_data = (const float *) src1->data; \ - DST_CTYPE * restrict dst_data = (DST_CTYPE *) dst->data; \ - dma_queue * dmaq = octx->ctx->dma[ith]; \ - uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; \ - uint8_t * dst_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; \ - float * srcb = (float *) src_base; \ - DST_CTYPE * dstb = (DST_CTYPE *) dst_base; \ - const uint32_t nrows = N * OH; \ - const uint32_t per_thread = ictx->pe_rows_per_thread; \ - const uint32_t row_start = per_thread * ith; \ - const uint32_t row_end = MIN(row_start + per_thread, nrows); \ - if (row_start >= row_end) \ - return; \ - for (uint32_t r = row_start; r < row_end; r++) { \ - const uint32_t in = r / OH; \ - const uint32_t ioh = r % OH; \ - for (uint32_t ikh = 0; ikh < KH; ikh++) { \ - int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh; \ - int ok = (iih >= 0 && iih < (int32_t) IH); \ - for (uint32_t iic = 0; iic < IC; iic++) { \ - float * vdst = srcb + ((uint64_t) (iic * KH + ikh)) * IW; \ - const float * _vsrc = \ - ok ? (src_data + ((uint64_t) (in * IC + iic) * IH + iih) * IW) : (const float *) vdst; \ - dma_queue_push_ddr_to_vtcm( \ - dmaq, dma_make_ptr((uint8_t *) vdst, ok ? (const uint8_t *) _vsrc : (const uint8_t *) vdst), \ - IW * sizeof(float), IW * sizeof(float), ok ? 1 : 0); \ - } \ - } \ - for (uint32_t i = 0; i < IC * KH; i++) \ - dma_queue_pop(dmaq); \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, r); \ - for (uint32_t iow = 0; iow < OW; iow++) { \ - DST_CTYPE * dst_patch = dstb + (uint64_t) iow * patch_stride; \ - for (uint32_t ikh = 0; ikh < KH; ikh++) { \ - int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh; \ - for (uint32_t iic = 0; iic < IC; iic++) { \ - DST_CTYPE * out_run = dst_patch + iic * (KH * KW) + ikh * KW; \ - if (iih < 0 || iih >= (int32_t) IH) { \ - SPLAT_FN(out_run, 0.0f, KW); \ - continue; \ - } \ - const float * src_run = srcb + ((uint64_t) (iic * KH + ikh)) * IW + (uint64_t) iow * KW; \ - COPY_FN((uint8_t *) out_run, (const uint8_t *) src_run, KW); \ - } \ - } \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, r); \ - DST_CTYPE * ddr_row = dst_data + ((uint64_t) (in * OH + ioh) * OW) * patch_stride; \ - dma_queue_push_vtcm_to_ddr(dmaq, dma_make_ptr((uint8_t *) ddr_row, (uint8_t *) dstb), \ - OW * patch_stride * (DST_ELEM), OW * patch_stride * (DST_ELEM), 1); \ - dma_queue_flush(dmaq); \ - } \ +// Software-pipelined 2-deep: while HVX computes block bi from buffer slot +// (bi&1), the DMA engine stages block bi+1 into the other slot concurrently. +// A single dma_queue_flush per iteration (after issuing the next stage-in and +// this block's store-out) waits for both - safe because the ring is strict +// FIFO and each buffer slot is only reused after its prior consumer (compute +// or store-out) already finished in program order. +#define IM2COL_BLOCKED_DMA_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN, DST_ELEM, TAG) \ + static void FNAME(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; \ + struct htp_ops_context * octx = ictx->octx; \ + struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; \ + const struct htp_tensor * restrict src1 = octx->src[1]; \ + const struct htp_tensor * restrict dst = octx->dst; \ + const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1]; \ + const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3]; \ + const int32_t d0 = octx->op_params[4], d1 = octx->op_params[5]; \ + const int32_t is_2D = octx->op_params[6] == 1; \ + const uint32_t N = is_2D ? src1->ne[3] : src1->ne[2]; \ + const uint32_t IC = is_2D ? src1->ne[2] : src1->ne[1]; \ + const uint32_t IH = is_2D ? src1->ne[1] : 1; \ + const uint32_t IW = src1->ne[0]; \ + const uint32_t KH = is_2D ? octx->src[0]->ne[1] : 1; \ + const uint32_t KW = octx->src[0]->ne[0]; \ + const uint32_t OH = is_2D ? dst->ne[2] : 1; \ + const uint32_t OW = dst->ne[1]; \ + const uint32_t owb = ictx->pe_owb, Wb = ictx->pe_wb; \ + const uint32_t patch_stride = IC * KH * KW; \ + const dma_addr_t src_data = src1->data; \ + const dma_addr_t dst_data = dst->data; \ + dma_queue * dmaq = octx->ctx->dma[ith]; \ + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; \ + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; \ + float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; \ + DST_CTYPE * dstb2[2] = { (DST_CTYPE *) dstb_base, (DST_CTYPE *) (dstb_base + ictx->pe_dst_row_bytes) }; \ + const uint32_t nrows = N * OH; \ + const uint32_t per_thread = ictx->pe_rows_per_thread; \ + const uint32_t row_start = per_thread * ith; \ + const uint32_t row_end = MIN(row_start + per_thread, nrows); \ + if (row_start >= row_end) \ + return; \ + const uint32_t nbpr = (OW + owb - 1) / owb; \ + const uint32_t nrows_local = row_end - row_start; \ + const uint32_t total_blocks = nrows_local * nbpr; \ + for (uint32_t bi = 0; bi < total_blocks; bi++) { \ + const uint32_t buf = bi & 1u; \ + float * srcb = srcb2[buf]; \ + DST_CTYPE * dstb = dstb2[buf]; \ + const uint32_t r = row_start + bi / nbpr; \ + const uint32_t in = r / OH; \ + const uint32_t ioh = r % OH; \ + const uint32_t c0 = (bi % nbpr) * owb; \ + const uint32_t nb = MIN(owb, OW - c0); \ + const int32_t win0 = (int32_t) c0 * s0 - p0; \ + if (bi == 0) { \ + /* prologue: stage block 0 and wait - nothing to overlap with yet */ \ + for (uint32_t ikh = 0; ikh < KH; ikh++) { \ + const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \ + if (iih < 0 || iih >= (int32_t) IH) \ + continue; \ + const int32_t lo = win0 < 0 ? -win0 : 0; \ + int32_t hi = (int32_t) IW - win0; \ + if (hi > (int32_t) Wb) \ + hi = (int32_t) Wb; \ + if (hi <= lo) \ + continue; \ + const uint32_t cpw = (uint32_t) (hi - lo); \ + float * vdst = srcb + (size_t) ikh * Wb + lo; \ + const dma_addr_t vsrc = src_data + (size_t) (((in * IC) * IH + iih) * IW + (win0 + lo)) * sizeof(float); \ + while (!dma_queue_push(dmaq, dma_make_data(vdst, vsrc), \ + (size_t) KH * Wb * sizeof(float), (size_t) IH * IW * sizeof(float), \ + cpw * sizeof(float), IC)) { \ + dma_queue_pop(dmaq); \ + } \ + } \ + dma_queue_flush(dmaq); \ + } \ + if (bi + 1 < total_blocks) { \ + /* prefetch: stage block bi+1 into the other slot; overlaps with this block's compute below */ \ + const uint32_t nbuf = 1u - buf; \ + float * nsrcb = srcb2[nbuf]; \ + const uint32_t nr = row_start + (bi + 1) / nbpr; \ + const uint32_t nin = nr / OH; \ + const uint32_t nioh = nr % OH; \ + const uint32_t nc0 = ((bi + 1) % nbpr) * owb; \ + const int32_t nwin0 = (int32_t) nc0 * s0 - p0; \ + for (uint32_t ikh = 0; ikh < KH; ikh++) { \ + const int32_t iih = (int32_t) nioh * s1 + (int32_t) ikh * d1 - p1; \ + if (iih < 0 || iih >= (int32_t) IH) \ + continue; \ + const int32_t lo = nwin0 < 0 ? -nwin0 : 0; \ + int32_t hi = (int32_t) IW - nwin0; \ + if (hi > (int32_t) Wb) \ + hi = (int32_t) Wb; \ + if (hi <= lo) \ + continue; \ + const uint32_t cpw = (uint32_t) (hi - lo); \ + float * vdst = nsrcb + (size_t) ikh * Wb + lo; \ + const dma_addr_t vsrc = src_data + (size_t) (((nin * IC) * IH + iih) * IW + (nwin0 + lo)) * sizeof(float); \ + while (!dma_queue_push(dmaq, dma_make_data(vdst, vsrc), \ + (size_t) KH * Wb * sizeof(float), (size_t) IH * IW * sizeof(float), \ + cpw * sizeof(float), IC)) { \ + dma_queue_pop(dmaq); \ + } \ + } \ + } \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, r); \ + for (uint32_t j = 0; j < nb; j++) { \ + const uint32_t iow = c0 + j; \ + DST_CTYPE * dst_patch = dstb + (uint64_t) j * patch_stride; \ + const int32_t iiw0 = (int32_t) iow * s0 - p0; \ + for (uint32_t ikh = 0; ikh < KH; ikh++) { \ + const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \ + const int okh = (iih >= 0 && iih < (int32_t) IH); \ + for (uint32_t iic = 0; iic < IC; iic++) { \ + DST_CTYPE * out_run = dst_patch + iic * (KH * KW) + ikh * KW; \ + if (!okh) { \ + SPLAT_FN(out_run, 0.0f, KW); \ + continue; \ + } \ + const float * vrow = srcb + ((uint64_t) (iic * KH + ikh)) * Wb; /* col win0 at idx 0*/ \ + if (d0 == 1) { \ + /* contiguous run within the staged window: [lo,hi) in-bounds, tails zero pad */ \ + const int32_t lo = iiw0 < 0 ? -iiw0 : 0; \ + int32_t hi = (int32_t) IW - iiw0; \ + if (hi > (int32_t) KW) { \ + hi = (int32_t) KW; \ + } \ + if (hi <= lo) { \ + SPLAT_FN(out_run, 0.0f, KW); \ + } else { \ + if (lo > 0) { \ + SPLAT_FN(out_run, 0.0f, (uint32_t) lo); \ + } \ + COPY_FN((uint8_t *) (out_run + lo), (const uint8_t *) (vrow + (iiw0 + lo - win0)), \ + (uint32_t) (hi - lo)); \ + if (hi < (int32_t) KW) { \ + SPLAT_FN(out_run + hi, 0.0f, (KW - (uint32_t) hi)); \ + } \ + } \ + continue; \ + } \ + for (uint32_t ikw = 0; ikw < KW; ikw++) { \ + const int32_t iiw = iiw0 + (int32_t) ikw * d0; \ + out_run[ikw] = \ + (iiw < 0 || iiw >= (int32_t) IW) ? (DST_CTYPE) 0.0f : (DST_CTYPE) vrow[iiw - win0]; \ + } \ + } \ + } \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, r); \ + const dma_addr_t ddr = dst_data + (size_t) ((in * OH + ioh) * OW + c0) * patch_stride * (DST_ELEM); \ + dma_queue_push(dmaq, dma_make_data(ddr, dstb), \ + nb * patch_stride * (DST_ELEM), nb * patch_stride * (DST_ELEM), \ + nb * patch_stride * (DST_ELEM), 1); \ + dma_queue_flush(dmaq); \ + } \ + } +IM2COL_BLOCKED_DMA_BODY(im2col_blocked_dma_thread, __fp16, hvx_copy_f16_f32_uu, hvx_splat_f16_u, sizeof(__fp16), "blk-dma-f16") +IM2COL_BLOCKED_DMA_BODY(im2col_blocked_dma_f32_thread, float, hvx_copy_f32_uu, hvx_splat_f32_u, sizeof(float), "blk-dma-f32") + +// Exact-tiling patch-embed DMA fast path (s0==KW, p0=0, d0=1; and 2D s1==KH, +// p1=0, d1=1). Intentionally reads no stride/pad/dilation params so the inner +// copy stays tight and fully hoisted - do NOT graft the general gather in here. +#define IM2COL_PATCHEMBED_DMA_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN, DST_ELEM, TAG) \ + static void FNAME(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; \ + struct htp_ops_context * octx = ictx->octx; \ + struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; \ + const struct htp_tensor * restrict src1 = octx->src[1]; \ + const struct htp_tensor * restrict dst = octx->dst; \ + const int32_t is_2D = octx->op_params[6] == 1; \ + const uint32_t N = is_2D ? src1->ne[3] : src1->ne[2]; \ + const uint32_t IC = is_2D ? src1->ne[2] : src1->ne[1]; \ + const uint32_t IH = is_2D ? src1->ne[1] : 1; \ + const uint32_t IW = src1->ne[0]; \ + const uint32_t KH = is_2D ? octx->src[0]->ne[1] : 1; \ + const uint32_t KW = octx->src[0]->ne[0]; \ + const uint32_t OH = is_2D ? dst->ne[2] : 1; \ + const uint32_t OW = dst->ne[1]; \ + const uint32_t patch_stride = IC * KH * KW; \ + const dma_addr_t src_data = src1->data; \ + const dma_addr_t dst_data = dst->data; \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; \ + uint8_t * dst_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; \ + float * srcb = (float *) src_base; \ + DST_CTYPE * dstb = (DST_CTYPE *) dst_base; \ + const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; \ + const uint32_t per_thread = ictx->pe_rows_per_thread; \ + const uint32_t row_start = ictx->pe_row_base + per_thread * ith; \ + const uint32_t row_end = MIN(row_start + per_thread, row_end_max); \ + if (row_start >= row_end) \ + return; \ + for (uint32_t r = row_start; r < row_end; r++) { \ + const uint32_t in = r / OH; \ + const uint32_t ioh = r % OH; \ + for (uint32_t ikh = 0; ikh < KH; ikh++) { \ + int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh; \ + int ok = (iih >= 0 && iih < (int32_t) IH); \ + for (uint32_t iic = 0; iic < IC; iic++) { \ + float * vdst = srcb + (size_t) (iic * KH + ikh) * IW; \ + const dma_addr_t vsrc = ok \ + ? (src_data + (size_t) ((in * IC + iic) * IH + iih) * IW * sizeof(float)) \ + : src_data; \ + dma_queue_push(dma_q, dma_make_data(vdst, vsrc), \ + IW * sizeof(float), IW * sizeof(float), IW * sizeof(float), ok ? 1 : 0); \ + } \ + } \ + for (uint32_t i = 0; i < IC * KH; i++) \ + dma_queue_pop(dma_q); \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, r); \ + for (uint32_t iow = 0; iow < OW; iow++) { \ + DST_CTYPE * dst_patch = dstb + (uint64_t) iow * patch_stride; \ + for (uint32_t ikh = 0; ikh < KH; ikh++) { \ + int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh; \ + for (uint32_t iic = 0; iic < IC; iic++) { \ + DST_CTYPE * out_run = dst_patch + iic * (KH * KW) + ikh * KW; \ + if (iih < 0 || iih >= (int32_t) IH) { \ + SPLAT_FN(out_run, 0.0f, KW); \ + continue; \ + } \ + const float * src_run = srcb + ((uint64_t) (iic * KH + ikh)) * IW + (uint64_t) iow * KW; \ + COPY_FN((uint8_t *) out_run, (const uint8_t *) src_run, KW); \ + } \ + } \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, r); \ + const dma_addr_t ddr_row = dst_data + (size_t) (in * OH + ioh) * OW * patch_stride * (DST_ELEM); \ + dma_queue_push(dma_q, dma_make_data(ddr_row, dstb), \ + OW * patch_stride * (DST_ELEM), OW * patch_stride * (DST_ELEM), \ + OW * patch_stride * (DST_ELEM), 1); \ + dma_queue_flush(dma_q); \ + } \ } IM2COL_PATCHEMBED_DMA_BODY(im2col_patchembed_dma_thread, __fp16, hvx_copy_f16_f32_uu, hvx_splat_f16_u, sizeof(__fp16), "pe-dma-f16") @@ -209,21 +390,30 @@ static bool im2col_use_patchembed_dma(const struct htp_ops_context * octx) { const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3]; const int32_t d0 = octx->op_params[4], d1 = octx->op_params[5]; const int is_2D = octx->op_params[6] == 1; - if (!is_2D) { - return false; - } if (octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_F32) { return false; } - const uint32_t KH = octx->src[0]->ne[1], KW = octx->src[0]->ne[0]; - if (s0 != (int32_t) KW || s1 != (int32_t) KH) { - return false; // non-overlapping + const uint32_t KH = is_2D ? octx->src[0]->ne[1] : 1; + const uint32_t KW = octx->src[0]->ne[0]; + if (s0 != (int32_t) KW) { + return false; // non-overlapping (width) + } + if (p0 != 0) { + return false; // no padding (width) } - if (p0 != 0 || p1 != 0) { - return false; // no padding + if (d0 != 1) { + return false; // no dilation (width) } - if (d0 != 1 || d1 != 1) { - return false; // no dilation + if (is_2D) { + if (s1 != (int32_t) KH) { + return false; // non-overlapping (height) + } + if (p1 != 0) { + return false; // no padding (height) + } + if (d1 != 1) { + return false; // no dilation (height) + } } return true; } @@ -233,8 +423,11 @@ static bool im2col_use_patchembed_dma(const struct htp_ops_context * octx) { static bool im2col_patchembed_dma_fits(struct htp_ops_context * octx, struct htp_im2col_context * ictx, uint32_t n_threads) { - const uint32_t IC = octx->src[1]->ne[2], IW = octx->src[1]->ne[0]; - const uint32_t KH = octx->src[0]->ne[1], KW = octx->src[0]->ne[0]; + const int32_t is_2D = octx->op_params[6] == 1; + const uint32_t IC = is_2D ? octx->src[1]->ne[2] : octx->src[1]->ne[1]; + const uint32_t IW = octx->src[1]->ne[0]; + const uint32_t KH = is_2D ? octx->src[0]->ne[1] : 1; + const uint32_t KW = octx->src[0]->ne[0]; const uint32_t OW = octx->dst->ne[1]; const uint32_t patch_stride = IC * KH * KW; @@ -257,6 +450,45 @@ static bool im2col_patchembed_dma_fits(struct htp_ops_context * octx, return true; } +// Sizes a per-thread 2x(src,dst) VTCM ping-pong for the blocked general kernel. +// Stages Wb=(owb-1)*s0+(KW-1)*d0+1 source cols per (iic,ikh) row and owb patches +// of dst. Picks the largest owb that fits; returns false if even owb=1 does not. +static bool im2col_blocked_dma_fits(struct htp_ops_context * octx, + struct htp_im2col_context * ictx, + uint32_t n_threads) { + const int32_t is_2D = octx->op_params[6] == 1; + const int32_t s0 = octx->op_params[0]; + const int32_t d0 = octx->op_params[4]; + const uint32_t IC = is_2D ? octx->src[1]->ne[2] : octx->src[1]->ne[1]; + const uint32_t KH = is_2D ? octx->src[0]->ne[1] : 1; + const uint32_t KW = octx->src[0]->ne[0]; + const uint32_t OW = octx->dst->ne[1]; + const uint32_t patch_stride = IC * KH * KW; + const uint32_t dst_elem = (octx->dst->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); + + for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) { + const uint32_t Wb = (owb - 1) * (uint32_t) s0 + (KW - 1) * (uint32_t) d0 + 1; + const uint32_t src_row_bytes = hex_round_up(IC * KH * Wb * sizeof(float), 256); + const uint32_t dst_row_bytes = hex_round_up(owb * patch_stride * dst_elem, 256); + struct htp_im2col_vtcm_layout L; + htp_im2col_vtcm_layout_build(&L, src_row_bytes, dst_row_bytes, n_threads); + if (L.total_bytes <= octx->ctx->vtcm_size) { + uint8_t * const base = octx->ctx->vtcm_base; + ictx->pe_owb = owb; + ictx->pe_wb = Wb; + ictx->pe_src_row_bytes = src_row_bytes; + ictx->pe_dst_row_bytes = dst_row_bytes; + ictx->pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src); + ictx->pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + ictx->pe_src_size_per_thread = (uint32_t) L.src_bytes_per_thread; + ictx->pe_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread; + return true; + } + if (owb == 1) break; // avoid unsigned underflow + } + return false; +} + int op_im2col(struct htp_ops_context * octx) { const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; @@ -266,35 +498,83 @@ int op_im2col(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t N = src1->ne[3]; - const uint32_t OH = dst->ne[2]; - const uint32_t OW = dst->ne[1]; - const uint32_t npatches = N * OH * OW; - const uint32_t n_threads = MIN(octx->n_threads, npatches); + const int32_t is_2D = octx->op_params[6] == 1; + const uint32_t N = is_2D ? src1->ne[3] : src1->ne[2]; + const uint32_t OH = is_2D ? dst->ne[2] : 1; + const uint32_t OW = dst->ne[1]; + const uint32_t total_patches = N * OH * OW; + const uint32_t total_rows = N * OH; + + uint32_t patch_base = 0; + uint32_t npatches = total_patches; + if (octx->ctx->mdev.count > 1) { + const uint32_t patch_size = dst->nb[1]; + const uint32_t patches_per_chunk = + (patch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(patch_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = + htp_tensor_mdev_partition(total_patches, htp_tensor_mdev_data_aligned(dst) ? patches_per_chunk : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + patch_base = range.start; + npatches = range.count; + } + + uint32_t row_base = 0; + uint32_t nrows = total_rows; + if (octx->ctx->mdev.count > 1) { + const uint32_t row_size = dst->nb[2]; + const uint32_t rows_per_chunk = + (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = + htp_tensor_mdev_partition(total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_base = range.start; + nrows = range.count; + } - if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) || n_threads == 0) { + if (npatches == 0 && nrows == 0) { return HTP_STATUS_OK; } + const uint32_t n_threads = MIN(octx->n_threads, MAX(npatches, 1)); + struct htp_im2col_context ictx = { 0 }; - ictx.octx = octx; - ictx.npatches_per_thread = (npatches + n_threads - 1) / n_threads; + ictx.octx = octx; + ictx.patch_base = patch_base; + ictx.npatches = npatches; + ictx.npatches_per_thread = (npatches + n_threads - 1) / n_threads; // Clean non-overlapping patch-embed -> DMA kernel (if it fits VTCM); - // everything else (padding/dilation/stride edges) -> pure-DDR kernel. - if (im2col_use_patchembed_dma(octx)) { - const uint32_t nrows = N * OH; - const uint32_t pth = MIN(octx->n_threads, nrows); - if (pth > 0 && im2col_patchembed_dma_fits(octx, &ictx, pth)) { - ictx.pe_rows_per_thread = (nrows + pth - 1) / pth; - if (dst->type == HTP_TYPE_F16) { - work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_thread, &ictx, pth); - } else { - work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_f32_thread, &ictx, pth); + // everything else (padding/dilation/stride edges) -> blocked-staging DMA + // kernel; if neither fits VTCM -> pure-DDR kernel. + if (nrows > 0) { + const uint32_t pth = MIN(octx->n_threads, nrows); + if (pth > 0) { + ictx.pe_row_base = row_base; + ictx.pe_nrows = nrows; + const bool exact = im2col_use_patchembed_dma(octx); + if (exact && im2col_patchembed_dma_fits(octx, &ictx, pth)) { + ictx.pe_rows_per_thread = (nrows + pth - 1) / pth; + work_queue_run(octx->ctx->work_queue, + dst->type == HTP_TYPE_F16 ? im2col_patchembed_dma_thread + : im2col_patchembed_dma_f32_thread, &ictx, pth); + return HTP_STATUS_OK; + } + if (!exact && im2col_blocked_dma_fits(octx, &ictx, pth)) { + ictx.pe_rows_per_thread = (nrows + pth - 1) / pth; + work_queue_run(octx->ctx->work_queue, + dst->type == HTP_TYPE_F16 ? im2col_blocked_dma_thread + : im2col_blocked_dma_f32_thread, &ictx, pth); + return HTP_STATUS_OK; } - return HTP_STATUS_OK; } - // else: doesn't fit -> fall through to the pure-DDR kernel below. + } + // Fall through to pure-DDR. + if (npatches == 0) { + return HTP_STATUS_OK; + } + + if (htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; } if (dst->type == HTP_TYPE_F16) { diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 3ab4613cf921..b4b352b206f6 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -21,7 +21,7 @@ #include <stdatomic.h> #include "hex-utils.h" -#include "hex-dma.h" +#include "dma-queue.h" #include "hmx-queue.h" #define GGML_COMMON_DECL_C @@ -34,8 +34,9 @@ #include "work-queue.h" #include "hex-profile.h" #include "allreduce-ops.h" +#include "htp-fence.h" -#define HMX_QUEUE_CAPACITY 16 +#define HMX_QUEUE_CAPACITY 128 #define HMX_QUEUE_STACK_SIZE 16384 #define WORK_QUEUE_CAPACITY 16 #define WORK_QUEUE_STACK_SIZE 16384 @@ -48,33 +49,74 @@ struct htp_handle { struct htp_context * ctx; }; -static inline void * htp_mmap(uint32_t fd, uint32_t size) { +static inline uint64_t htp_mmap(uint32_t fd, uint64_t size, uint32_t flags) { +#if __HVX_ARCH__ > 79 + if (flags & HTP_BUF_EXTENDED) { + HAP_mem_req_payload_t payload; + memset(&payload, 0, sizeof(payload)); + payload.request_id = HAP_MEM_MAP; + payload.mmap.len = size; + payload.mmap.prot = HAP_MEM_CACHE_NON_SHARED | HAP_PROT_READ; + payload.mmap.flags = HAP_MEM_FLAGS_EXTENDED_MAP; + payload.mmap.fd = fd; + + if (HAP_mem_request(&payload) != 0) { + FARF(ERROR, "extended mmap failed : fd %u size %llu", fd, (unsigned long long) size); + return 0; + } + + return payload.mmap.dsp_va; + } +#else + if (flags & HTP_BUF_EXTENDED) { + FARF(ERROR, "extended mmap is unsupported on v%d", __HVX_ARCH__); + return 0; + } +#endif + + if (size > UINT32_MAX) { + FARF(ERROR, "mmap failed : size %llu exceeds 32-bit limit", (unsigned long long) size); + return 0; + } + void * va = (void *)-1; for (int retry = 0; retry < 2; retry++) { #if __HVX_ARCH__ > 73 - va = HAP_mmap2(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); + va = HAP_mmap2(NULL, (size_t) size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); #else if (size > HTP_MMAP_MAX_VMEM) { - FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) size); + FARF(ERROR, "mmap failed : size %llu exceeds 2GB limit for HAP_mmap", (unsigned long long) size); abort(); } - va = HAP_mmap(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); + va = HAP_mmap(NULL, (int) size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); #endif if (va != (void *)-1 && va != NULL) { - return va; + return (uint64_t) (uintptr_t) va; } if (retry == 0) { - FARF(HIGH, "mmap failed first try (va %p fd %u size %u), retrying...", va, fd, size); + FARF(HIGH, "mmap failed first try (va %p fd %u size %llu), retrying...", va, fd, (unsigned long long) size); } } - return NULL; + return 0; } -static inline void htp_munmap(void * va, uint32_t size) { +static inline void htp_munmap(uint64_t va, uint64_t size, uint32_t flags) { +#if __HVX_ARCH__ > 79 + if (flags & HTP_BUF_EXTENDED) { + HAP_mem_req_payload_t payload; + memset(&payload, 0, sizeof(payload)); + payload.request_id = HAP_MEM_UNMAP; + payload.munmap.dsp_va = va; + payload.munmap.len = size; + HAP_mem_request(&payload); + return; + } +#endif + #if __HVX_ARCH__ > 73 - HAP_munmap2(va, size); + HAP_munmap2((void *) (uintptr_t) va, (size_t) size); #else - HAP_munmap(va, size); + HAP_munmap((void *) (uintptr_t) va, (int) size); #endif } @@ -159,10 +201,11 @@ AEEResult htp_iface_close(remote_handle64 handle) { // release the mmaps (if any) for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) { if (ctx->mmap[i].size) { - htp_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size); + htp_munmap(ctx->mmap[i].base, ctx->mmap[i].size, ctx->mmap[i].flags); ctx->mmap[i].size = 0; - ctx->mmap[i].base = NULL; + ctx->mmap[i].base = 0; ctx->mmap[i].fd = -1; + ctx->mmap[i].flags = 0; } } @@ -183,7 +226,7 @@ AEEResult htp_iface_close(remote_handle64 handle) { return AEE_SUCCESS; } -AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) { +AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint64_t size) { struct htp_handle * h = (struct htp_handle *) handle; if (!h || !h->ctx) { return AEE_EBADPARM; @@ -202,16 +245,17 @@ AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) { for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) { struct htp_mmap *m = &ctx->mmap[i]; if (!m->size) { - FARF(HIGH, "mmap : fd %u size %u", fd, size); - void *va = htp_mmap(fd, size); - if (va == NULL) { - FARF(ERROR, "mmap failed : fd %u size %u", fd, (uint32_t) size); + FARF(HIGH, "mmap : fd %u size %llu", fd, (unsigned long long) size); + uint64_t va = htp_mmap(fd, size, 0); + if (va == 0) { + FARF(ERROR, "mmap failed : fd %u size %llu", fd, (unsigned long long) size); return AEE_EFAILED; } - m->base = (uint64_t) va; + m->base = va; m->fd = fd; m->size = size; + m->flags = 0; return AEE_SUCCESS; } @@ -230,11 +274,12 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) { for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) { struct htp_mmap *m = &ctx->mmap[i]; if (fd < 0 || m->fd == fd) { - FARF(HIGH, "unmmap : base %p fd %u size %u", (void*) m->base, m->fd, (uint32_t) m->size); - htp_munmap((void *) m->base, m->size); + FARF(HIGH, "unmmap : base 0x%llx fd %u size %llu", (unsigned long long) m->base, m->fd, (unsigned long long) m->size); + htp_munmap(m->base, m->size, m->flags); m->size = 0; m->base = NULL; m->fd = -1; + m->flags = 0; } } @@ -393,8 +438,6 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp for (uint32_t i = 0; i < n_hvx; i++) { size_dma = hex_align_up(size_dma, dma_queue_alignof()); size_dma += dma_queue_sizeof(256); - size_dma = hex_align_up(size_dma, dma_queue_alignof()); - size_dma += dma_queue_alias_sizeof(); } offset = offset_dma + size_dma; @@ -537,16 +580,11 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp // Initialize DMA queues uint8_t * dma_ptr_curr = (uint8_t *) ((uintptr_t) block + offset_dma); size_t size_dma_q = dma_queue_sizeof(256); - size_t size_dma_alias = dma_queue_alias_sizeof(); for (int i = 0; i < ctx->n_threads; i++) { dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof()); - ctx->dma_cached[i] = dma_queue_init(dma_ptr_curr, 256, (uintptr_t) ctx->vtcm_base, ctx->vtcm_size, &ctx->trace[i]); + ctx->dma[i] = dma_queue_init(dma_ptr_curr, 256, &ctx->trace[i]); dma_ptr_curr += size_dma_q; - - dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof()); - ctx->dma[i] = dma_queue_alias_init(dma_ptr_curr, ctx->dma_cached[i], 1); - dma_ptr_curr += size_dma_alias; } ctx->ddr_spad_size = 512 * 1024; // 512 KB @@ -607,8 +645,7 @@ AEEResult htp_iface_stop(remote_handle64 handle) { work_queue_free(ctx->work_queue); for (int i = 0; i < ctx->n_threads; i++) { - dma_queue_alias_free(ctx->dma[i]); - dma_queue_free(ctx->dma_cached[i]); + dma_queue_free(ctx->dma[i]); } if (ctx->hmx_queue) { @@ -710,22 +747,43 @@ static inline void profile_stop(uint32_t mode, struct profile_data * d) { static int op_fence(struct htp_ops_context * octx) { struct htp_context *ctx = octx->ctx; struct htp_thread_trace * tr = &ctx->trace[0]; - const uint32_t seq = (uint32_t) octx->op_params[0]; + const uint32_t seq = (uint32_t) octx->op_params[0]; + const uint32_t mode = (uint32_t) octx->op_params[1]; htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); const struct htp_tensor * sync = octx->src[0]; - atomic_uint * sync_fence = (atomic_uint *) sync->data; + atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data; + + if (mode == 1) { + htp_flush_dirty_ranges(ctx); + + htp_mdev_group_barrier(octx); + + if (ctx->mdev.idx == 0) { + htp_fence_write(sync_fence, seq, octx->status); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + FARF(HIGH, "ggml-hex: sync-signal : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status); + return octx->status; + } + + int status = HTP_STATUS_OK; uint64_t spins = 0; while (1) { - Q6_dccleaninva_A((void *) sync_fence); - asm volatile ("syncht" : : : "memory"); - uint32_t val = atomic_load(&sync_fence[0]); - if ((int32_t)(val - seq) >= 0) { + uint32_t sync_seq; + uint32_t sync_status; + htp_fence_read(sync_fence, &sync_seq, &sync_status); + if ((int32_t)(sync_seq - seq) >= 0) { + if (sync_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: sync-wait peer failed with status %u : fence %p seq 0x%x\n", sync_status, sync_fence, seq); + status = sync_status; + } break; } if (++spins > HTP_FENCE_TIMEOUT) { - FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq %u\n", sync_fence, spins, seq); + FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq); + status = HTP_STATUS_INTERNAL_ERR; break; } hex_pause(); @@ -733,12 +791,27 @@ static int op_fence(struct htp_ops_context * octx) { htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); - FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq %u\n", sync_fence, spins, seq); + FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq); + return status; +} + +static int op_mdev_group(struct htp_ops_context * octx) { + struct htp_context * ctx = octx->ctx; + const struct htp_tensor * sync = octx->src[0]; + ctx->mdev.idx = (uint16_t) octx->op_params[0]; + ctx->mdev.count = (uint16_t) sync->ne[1]; + if (ctx->mdev.count > 1) { + ctx->mdev.count_div = init_fastdiv_values(ctx->mdev.count); + ctx->mdev.fence_base = (uint8_t *) sync->data; + } return HTP_STATUS_OK; } static int execute_op(struct htp_ops_context * octx) { switch (octx->op) { + case HTP_OP_MDEV_GROUP: + return op_mdev_group(octx); + case HTP_OP_FENCE: return op_fence(octx); @@ -771,6 +844,7 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_RMS_NORM_MUL: case HTP_OP_SCALE: case HTP_OP_CLAMP: + case HTP_OP_LEAKY_RELU: case HTP_OP_SQR: case HTP_OP_SQRT: case HTP_OP_UNARY_SOFTPLUS: @@ -782,6 +856,7 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_UNARY_TANH: case HTP_OP_UNARY_ABS: case HTP_OP_UNARY_LOG: + case HTP_OP_UNARY_RELU: case HTP_OP_L2_NORM: return op_unary(octx); @@ -789,6 +864,7 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_GLU_SWIGLU_OAI: case HTP_OP_GLU_SWIGLU_CLAMP: case HTP_OP_GLU_GEGLU: + case HTP_OP_GLU_GEGLU_QUICK: return op_activations(octx); case HTP_OP_SOFTMAX: @@ -810,6 +886,7 @@ static int execute_op(struct htp_ops_context * octx) { return op_sum_rows(octx); case HTP_OP_CPY: + case HTP_OP_CPY_FENCE: return op_cpy(octx); case HTP_OP_REPEAT: @@ -818,6 +895,9 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_ARGSORT: return op_argsort(octx); + case HTP_OP_TOP_K: + return op_top_k(octx); + case HTP_OP_SSM_CONV: return op_ssm_conv(octx); @@ -839,6 +919,9 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_IM2COL: return op_im2col(octx); + case HTP_OP_ROLL: + return op_roll(octx); + case HTP_OP_CONCAT: return op_concat(octx); @@ -853,7 +936,7 @@ static int execute_op(struct htp_ops_context * octx) { } FARF(ERROR, "Unknown Op %u", octx->op); - return -1; + return HTP_STATUS_NO_SUPPORT; } static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct htp_buf_desc *b) { @@ -861,7 +944,7 @@ static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) { struct htp_mmap *m = ctx->mmap + i; - if (m->size && m->fd == b->fd) { + if (m->size && m->fd == b->fd && m->flags == b->flags) { b->base = m->base; *m_reuse |= (1 << i); return true; @@ -873,11 +956,12 @@ static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct static inline void drop_mmap(struct htp_context *ctx, struct htp_mmap *m) { if (m->size) { - FARF(ALWAYS, "unmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size); - htp_munmap((void *) m->base, m->size); + FARF(ALWAYS, "unmap : fd %u base 0x%llx size %llu", m->fd, (unsigned long long) m->base, (unsigned long long) m->size); + htp_munmap(m->base, m->size, m->flags); m->size = 0; m->base = 0; m->fd = -1; + m->flags = 0; } } @@ -888,17 +972,18 @@ static inline bool mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) { for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) { struct htp_mmap *m = &ctx->mmap[i]; if (!m->size) { - void *va = htp_mmap(b->fd, b->size); - if (va == NULL) { - FARF(HIGH, "mmap failed (will attempt defrag) : fd %u size %u", b->fd, (uint32_t) b->size); + uint64_t va = htp_mmap(b->fd, b->size, b->flags); + if (va == 0) { + FARF(HIGH, "mmap failed (will attempt defrag) : fd %u size %llu", b->fd, (unsigned long long) b->size); return false; } - m->base = b->base = (uint64_t) va; + m->base = b->base = va; m->fd = b->fd; m->size = b->size; + m->flags = b->flags; - FARF(ALWAYS, "mmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size); + FARF(ALWAYS, "mmap : fd %u base 0x%llx size %llu flags 0x%x", m->fd, (unsigned long long) m->base, (unsigned long long) m->size, m->flags); return true; } } @@ -917,14 +1002,22 @@ static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uin // See what we can reuse for (uint32_t i=0; i < n_bufs; i++) { struct htp_buf_desc *b = bufs + i; - if (reuse_buf(ctx, &m_reuse, b)) { b_reuse++; } else { e_vmem += b->size; } - FARF(HIGH, "prep-buf #%u : pass0 fd %u base %p size %u flags 0x%x", i, b->fd, (void*) b->base, (uint32_t) b->size, b->flags); + if (reuse_buf(ctx, &m_reuse, b)) { + b_reuse++; + } else if (!(b->flags & HTP_BUF_EXTENDED)) { + e_vmem += b->size; + } + FARF(HIGH, "prep-buf #%u : pass0 fd %u base 0x%llx size %llu flags 0x%x", i, b->fd, (unsigned long long) b->base, (unsigned long long) b->size, b->flags); } if (b_reuse == n_bufs) return; // all bufs reuse existing mappings // See how much vmem we have mmaped right now - for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) { m_vmem += ctx->mmap[i].size; } + for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) { + if (!(ctx->mmap[i].flags & HTP_BUF_EXTENDED)) { + m_vmem += ctx->mmap[i].size; + } + } FARF(HIGH, "prep-bufs : pass1 mmap-vmem %zu extra-vmem %zu max-vmem %zu : n-bufs %u b-reuse %u", (size_t) m_vmem, (size_t) e_vmem, (size_t) ctx->max_vmem, n_bufs, b_reuse); @@ -933,7 +1026,9 @@ static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uin // Drop unused mappings for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) { bool used = m_reuse & (1<<i); - if (!used) { drop_mmap(ctx, ctx->mmap + i); } + if (!used && !(ctx->mmap[i].flags & HTP_BUF_EXTENDED)) { + drop_mmap(ctx, ctx->mmap + i); + } } } @@ -945,35 +1040,40 @@ static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uin mmap_ok = false; break; } - FARF(HIGH, "prep-buf #%u : pass1 fd %u base %p size %u flags 0x%x", i, b->fd, (void*) b->base, (uint32_t) b->size, b->flags); + FARF(HIGH, "prep-buf #%u : pass1 fd %u base 0x%llx size %llu flags 0x%x", i, b->fd, (unsigned long long) b->base, (unsigned long long) b->size, b->flags); } if (!mmap_ok) { - // Attempt clean defragmentation: drop all mappings and remap (pass 2) - FARF(HIGH, "prep-bufs : dropping all mappings to defragment address space"); - for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) { drop_mmap(ctx, ctx->mmap + i); } + // Attempt defragmentation: drop 32-bit mappings and remap (pass 2) + FARF(HIGH, "prep-bufs : dropping 32-bit mappings to defragment address space"); + for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) { + if (!(ctx->mmap[i].flags & HTP_BUF_EXTENDED)) { + drop_mmap(ctx, ctx->mmap + i); + } + } for (uint32_t i=0; i < n_bufs; i++) { struct htp_buf_desc *b = bufs + i; - b->base = 0; + if (!(b->flags & HTP_BUF_EXTENDED)) { + b->base = 0; + } if (!mmap_buf(ctx, b)) { - FARF(ERROR, "prep-bufs : mmap failed after defragmentation (fd %u size %u)", b->fd, (uint32_t) b->size); + FARF(ERROR, "prep-bufs : mmap failed after defragmentation (fd %u size %llu)", b->fd, (unsigned long long) b->size); abort(); } - FARF(HIGH, "prep-buf #%u : pass2 fd %u base %p size %u flags 0x%x", i, b->fd, (void*) b->base, (uint32_t) b->size, b->flags); + FARF(HIGH, "prep-buf #%u : pass2 fd %u base 0x%llx size %llu flags 0x%x", i, b->fd, (unsigned long long) b->base, (unsigned long long) b->size, b->flags); } } } static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t idx, struct htp_tensor *t) { - uint32_t offset = t->data; - uint32_t size = t->size; + uint64_t offset = t->data; uint32_t bi = t->bi; - t->data = (uint32_t) (bufs[bi].base + offset); // update data to the actual pointer + t->data = bufs[bi].base + offset; // update data to the actual pointer - FARF(HIGH, "prep-tensor #%u: bi %u offset %u size %u data %p : %u:%u:%u:%u", idx, t->bi, offset, t->size, (void*) t->data, - t->ne[0], t->ne[1], t->ne[3], t->ne[3]); + FARF(HIGH, "prep-tensor #%u: bi %u offset %llu size %u data 0x%llx : %u:%u:%u:%u", idx, t->bi, (unsigned long long) offset, t->size, (unsigned long long) t->data, + t->ne[0], t->ne[1], t->ne[2], t->ne[3]); } static void prep_tensors(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t n_tens) { @@ -982,11 +1082,19 @@ static void prep_tensors(struct htp_context *ctx, struct htp_buf_desc *bufs, str } } -static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, uint32_t idx, struct htp_op_desc * op) { - memcpy(octx->op_params, op->params, sizeof(octx->op_params)); +static void mdev_group_init(struct htp_context * ctx, const struct htp_opbatch_req * req) { + memset(&ctx->mdev, 0, sizeof(ctx->mdev)); + ctx->mdev.fence_seq = (uint32_t)((req->seq & 0xfffff) << 12); +} + +static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs, uint32_t n_bufs, + struct htp_tensor * tens, uint32_t idx, struct htp_op_desc * op) { + memcpy(octx->op_params, op->params, sizeof(octx->op_params)); memcpy(octx->kernel_params, op->kernel_params, sizeof(octx->kernel_params)); - octx->flags = op->flags; - octx->op = op->opcode; + octx->flags = op->flags; + octx->op = op->opcode; + octx->n_threads = octx->ctx->n_threads; + octx->n_threads_div = octx->ctx->n_threads_div; FARF(HIGH, "proc-op #%u: opcode %u flags 0x%x", idx, octx->op, octx->flags); @@ -995,15 +1103,13 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u uint16_t src_idx = op->src[i]; if (src_idx == 0xffff) { octx->src[i] = NULL; - octx->src_dma[i] = NULL; continue; } struct htp_tensor *src = tens + src_idx; octx->src[i] = src; - octx->src_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma; - FARF(HIGH, "prep-src #%u: data %p size %u : %u:%u:%u:%u", op->src[i], (void*) src->data, src->size, + FARF(HIGH, "prep-src #%u: data 0x%llx size %u : %u:%u:%u:%u", op->src[i], (unsigned long long) src->data, src->size, src->ne[0], src->ne[1], src->ne[2], src->ne[3]); } @@ -1014,20 +1120,22 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u uint16_t dst_idx = op->dst[i]; if (dst_idx == 0xffff) { octx->dsts[i] = NULL; - octx->dst_dma[i] = NULL; continue; } struct htp_tensor *dst = tens + dst_idx; octx->dsts[i] = dst; - octx->dst_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma; - FARF(HIGH, "prep-dst[%u] #%u: data %p size %u : %u:%u:%u:%u", i, dst_idx, (void*) dst->data, dst->size, + FARF(HIGH, "prep-dst[%u] #%u: data 0x%llx size %u : %u:%u:%u:%u", i, dst_idx, (unsigned long long) dst->data, dst->size, dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } + htp_tensor_dirty_all(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS); + + htp_mdev_group_barrier(octx); + int status = execute_op(octx); - htp_tensor_dirty_all(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS); + htp_ops_context_set_status(octx, status); octx->src0_spad.src = NULL; octx->src1_spad.src = NULL; @@ -1035,7 +1143,7 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u octx->src3_spad.src = NULL; octx->dst_spad.src = NULL; - return status; + return octx->status; } static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_req * req, const struct dspqueue_buffer * dbuf) { @@ -1057,7 +1165,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r return; } - FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req->id, + FARF(HIGH, "processing opbatch #%llu: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", (unsigned long long) req->seq, n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size); // Setup descriptor pointers @@ -1094,8 +1202,11 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r struct htp_ops_context *octx = &ctx->octx; memset(octx, 0, sizeof(*octx)); - octx->n_threads = ctx->n_threads; - octx->ctx = ctx; + octx->n_threads = ctx->n_threads; + octx->n_threads_div = ctx->n_threads_div; + octx->ctx = ctx; + + mdev_group_init(ctx, req); work_queue_wakeup(ctx->work_queue); if (ctx->hmx_queue) { @@ -1103,15 +1214,18 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r } int op_status = HTP_STATUS_OK; - for (uint32_t i = 0; i < n_ops && op_status == HTP_STATUS_OK; i++) { + octx->status = HTP_STATUS_OK; + for (uint32_t i = 0; i < n_ops; i++) { struct profile_data prof; profile_start(ctx->profiler, &prof); - op_status = proc_op_req(octx, tens, i, &ops[i]); + op_status = proc_op_req(octx, bufs, n_bufs, tens, i, &ops[i]); profile_stop(ctx->profiler, &prof); + htp_ops_context_set_status(octx, op_status); + if (ctx->profiler) { pds[i].opcode = ops[i].opcode; pds[i].usecs = prof.usecs; @@ -1134,19 +1248,20 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); + htp_mdev_group_barrier(octx); + profile_stop(HTP_PROF_BASIC, &batch_prof); struct htp_opbatch_rsp rsp; memset(&rsp, 0, sizeof(rsp)); - rsp.id = req->id; - rsp.status = op_status; + rsp.seq = req->seq; + rsp.status = octx->status; rsp.n_bufs = n_bufs; rsp.n_tensors = n_tens; rsp.n_ops = n_ops; rsp.usecs = batch_prof.usecs; rsp.cycles_start = batch_prof.cycles_start; rsp.cycles_stop = batch_prof.cycles_stop; - rsp.seq = req->seq; if (ctx->profiler == HTP_PROF_TRACE) { for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index 2a87dd19ee8c..a09bc7a28317 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -11,7 +11,7 @@ #include <string.h> #include <stdatomic.h> -#include "hex-dma.h" +#include "dma-queue.h" #include "hvx-utils.h" #include "hvx-dump.h" #include "hvx-arith.h" @@ -21,25 +21,17 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "matmul-ops.h" #include "htp-vtcm.h" -static void hvx_tensor_add_f32_grid( - const struct htp_tensor * restrict dst, - const struct htp_tensor * restrict src2, - uint32_t start_row, - uint32_t end_row, - uint32_t start_col, - uint32_t end_col, - const struct fastdiv_values * div_ne11_12, - const struct fastdiv_values * div_ne11 -); - typedef struct { float *dst; - const float *src2; + dma_addr_t src2_addr; + size_t src2_bytes; const float *activation; - const __fp16 *weight; + dma_addr_t weight; + dma_queue * weight_dma; int m; int k; int n; @@ -65,6 +57,15 @@ typedef struct { struct fastdiv_values div_r3; } hmx_mm_f16_f32_batched_params_t; +static bool htp_matmul_has_extended_weight(const struct htp_ops_context * octx, uint32_t n_weights) { + for (uint32_t i = 0; i < n_weights; ++i) { + if (htp_tensor_is_extended(octx->src[i])) { + return true; + } + } + return false; +} + struct htp_mm_context { const char * type; struct htp_ops_context * octx; @@ -89,8 +90,12 @@ struct htp_mm_context { // Precomputed values uint32_t src0_nrows_per_thread; + uint32_t src0_row_start; + uint32_t src0_row_end; uint32_t src0_row_size_padded; uint32_t src1_nrows; + uint32_t cur_m_start; + uint32_t cur_m_rows; struct fastdiv_values mm_div_ne12_ne1; struct fastdiv_values mm_div_ne1; @@ -135,6 +140,23 @@ struct htp_mm_context { uint32_t vtcm_dst_size_per_thread; }; +static int htp_mm_init_context( + struct htp_ops_context * octx, + const struct htp_mm_kernel_params * kparams +) { + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (kparams->n_hmx) { + if (kparams->n_act_threads <= 0 || kparams->n_act_threads > (int32_t) octx->n_threads) { + return HTP_STATUS_INVAL_PARAMS; + } + } + + return HTP_STATUS_OK; +} + // vdelta control to expand first 32 e8m0 values into 32 uint32 elements static const uint8_t __attribute__((aligned(128))) expand_x32_e8m0[128] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x02, 0x00, 0x08, 0x08, 0x01, 0x02, 0x00, 0x04, 0x04, 0x00, 0x00, @@ -208,7 +230,7 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = { #define htp_matmul_preamble \ struct htp_mm_context * mmctx = data; \ struct htp_ops_context * octx = mmctx->octx; \ - dma_queue *dma_queue = octx->ctx->dma[ith]; \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread; \ htp_matmul_tensors_preamble; @@ -224,288 +246,213 @@ static inline void hvx_mm_run_quant_task(struct htp_mm_context * mmctx, unsigned } } -// *** matmul with support for 4d tensors and full broadcasting - -static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { - htp_matmul_preamble; - - assert(ne12 % ne02 == 0); - assert(ne13 % ne03 == 0); - - // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) - const uint32_t nr0 = ne0; - - // This is the size of the rest of the dimensions of the result - const uint32_t nr1 = ne1 * ne2 * ne3; - - // distribute the thread work across the inner or outer loop based on which one is larger - uint32_t dr0, dr1, ith0, ith1; - if (nr0 > nr1) { - dr0 = fastdiv(nr0 + nth - 1, &octx->ctx->n_threads_div); - dr1 = nr1; - ith0 = ith; - ith1 = 0; - } else { - dr0 = nr0; - dr1 = fastdiv(nr1 + nth - 1, &octx->ctx->n_threads_div); - ith0 = 0; - ith1 = ith; - } - const uint32_t ir0_start = dr0 * ith0; - const uint32_t ir0_end = MIN(ir0_start + dr0, nr0); - const uint32_t ir1_start = dr1 * ith1; - const uint32_t ir1_end = MIN(ir1_start + dr1, nr1); - - // no work for this thread - if (ir0_start >= ir0_end || ir1_start >= ir1_end) { - return; - } - - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0_start); - - const uint32_t blck_0 = 64; - const uint32_t blck_1 = 64; - - for (uint32_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { - for (uint32_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { - for (uint32_t ir1 = iir1; ir1 < MIN(iir1 + blck_1, ir1_end); ir1++) { - const uint32_t i13 = fastdiv(ir1, &mmctx->mm_div_ne12_ne1); - const uint32_t i12 = fastdiv(ir1 - i13 * ne12 * ne1, &mmctx->mm_div_ne1); - const uint32_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1); - - // broadcast src0 into src1 - const uint32_t i03 = fastdiv(i13, &mmctx->mm_div_r3); - const uint32_t i02 = fastdiv(i12, &mmctx->mm_div_r2); - - const uint32_t i1 = i11; - const uint32_t i2 = i12; - const uint32_t i3 = i13; - - const uint8_t * restrict src0_base = (const uint8_t *) src0->data + (0 + i02 * nb02 + i03 * nb03); - const uint8_t * restrict src1_col = (const uint8_t *) src1->data + (i11 * nb11 + i12 * nb12 + i13 * nb13); - float * dst_col = (float *) ((uint8_t * restrict) dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3)); - - const uint32_t ir0_block_end = MIN(iir0 + blck_0, ir0_end); - for (uint32_t ir0 = iir0; ir0 < ir0_block_end; ir0++) { - const uint8_t * restrict src0_row = src0_base + ir0 * nb01; - mmctx->vec_dot_1x1(ne00, &dst_col[ir0], src0_row, src1_col); - } - } - } - } - - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0_start); - if (src2) { - hvx_tensor_add_f32_grid(dst, src2, ir1_start, ir1_end, ir0_start, ir0_end, &mmctx->mm_div_ne12_ne1, &mmctx->mm_div_ne1); - } -} - -#include "hmx-mm-kernels-tiled.h" +// hvx kernels first: the HMX Q6_K dequantizer reuses unpack_q6_k_group from there #include "hvx-mm-kernels-tiled.h" -#include "hvx-mm-kernels-flat.h" +#include "hvx-mm-kernels-float.h" +#include "hmx-mm-kernels-tiled.h" // Specialized repacked matmul macros -#define MATMUL_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \ -static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ - htp_matmul_preamble; \ - \ - const uint32_t src0_nrows = ne01 * ne02 * ne03; \ - const uint32_t src1_nrows = ne11 * ne12 * ne13; \ - \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ - \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - \ - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ - const uint32_t n_prefetch = kparams->n_prefetch; \ - assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \ - \ - const size_t dst_row_size = nb1; \ - const size_t src1_row_size = nb11; \ - const size_t src1_stride = mmctx->vtcm_src1_stride; \ - const size_t src2_stride = src2 ? ((src2->ne[1] == 1) ? 0 : src2->nb[1]) : 0; \ - \ - uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; \ - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ - uint8_t * restrict src1_data = mmctx->vtcm_src1; \ - \ - const uint8_t * restrict src0_row = (const uint8_t *) src0->data; \ - \ - const uint32_t tile_size = TILE_SIZE; \ - const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \ - \ - uint32_t n_k_tiles_w = ne00 / 32; \ - uint32_t n_k_tiles_a = ne10 / 32; \ - uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ - uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ - \ - uint32_t ct_start = src0_start_row / 32; \ - uint32_t ct_end = (src0_end_row + 31) / 32; \ - \ - uint32_t push_ct = ct_start; \ - if (src0_start_row < src0_end_row) { \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ - src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - } \ - } \ - \ - hvx_mm_run_quant_task(mmctx, ith); \ - \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ - const uint8_t * w_tile = dma_queue_pop(dma_queue).dst; \ - \ - int valid_rows = (int)ne0 - (int)(ct * 32); \ - valid_rows = MIN(32, MAX(0, valid_rows)); \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - uint32_t ir1 = 0; \ - for (; ir1 + 1 < src1_nrows; ir1 += 2) { \ - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); \ - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); \ - float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size)); \ - float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size)); \ - \ - float * dst_ptr0 = &dst_row0[ct * 32]; \ - float * dst_ptr1 = &dst_row1[ct * 32]; \ - \ - const float * src2_ptr0 = NULL; \ - const float * src2_ptr1 = NULL; \ - if (src2) { \ - const float * restrict src2_row0 = (const float *) ((const uint8_t *) src2->data + ((ir1+0) * src2_stride)); \ - const float * restrict src2_row1 = (const float *) ((const uint8_t *) src2->data + ((ir1+1) * src2_stride)); \ - src2_ptr0 = &src2_row0[ct * 32]; \ - src2_ptr1 = &src2_row1[ct * 32]; \ - } \ - DOT_2X2(ne10, dst_ptr0, dst_ptr1, w_tile, src1_col0, src1_col1, valid_rows, src2_ptr0, src2_ptr1); \ - } \ - \ - for (; ir1 < src1_nrows; ++ir1) { \ - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); \ - float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); \ - float * dst_ptr = &dst_row[ct * 32]; \ - \ - const float * src2_ptr = NULL; \ - if (src2) { \ - const float * restrict src2_row = (const float *) ((const uint8_t *) src2->data + (ir1 * src2_stride)); \ - src2_ptr = &src2_row[ct * 32]; \ - } \ - DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, src2_ptr); \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - \ - if (push_ct < ct_end) { \ - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile, src0_row + push_ct * tile_row_stride), \ - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - push_ct++; \ - } \ - } \ +#define MATMUL_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \ +static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ + htp_matmul_preamble; \ + \ + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \ + const uint32_t src1_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); \ + const uint32_t cur_m_start = mmctx->cur_m_start; \ + \ + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \ + \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + \ + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ + const uint32_t n_prefetch = kparams->n_prefetch; \ + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \ + \ + const size_t dst_row_size = nb1; \ + const size_t src1_row_size = nb11; \ + const size_t src1_stride = mmctx->vtcm_src1_stride; \ + const size_t src2_stride = src2 ? ((src2->ne[1] == 1) ? 0 : src2->nb[1]) : 0; \ + \ + uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; \ + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ + uint8_t * restrict src1_data = mmctx->vtcm_src1; \ + \ + const dma_addr_t src0_row = src0->data; \ + \ + const uint32_t tile_size = TILE_SIZE; \ + const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \ + \ + uint32_t n_k_tiles_w = ne00 / 32; \ + uint32_t n_k_tiles_a = ne10 / 32; \ + uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ + uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ + \ + uint32_t ct_start = src0_start_row / 32; \ + uint32_t ct_end = (src0_end_row + 31) / 32; \ + \ + uint32_t push_ct = ct_start; \ + if (src0_start_row < src0_end_row) { \ + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ + src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + } \ + } \ + \ + hvx_mm_run_quant_task(mmctx, ith); \ + \ + if (src0_start_row >= src0_end_row) { \ + return; \ + } \ + \ + for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; \ + \ + int valid_rows = (int)ne0 - (int)(ct * 32); \ + valid_rows = MIN(32, MAX(0, valid_rows)); \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ + uint32_t ir1 = 0; \ + for (; ir1 + 1 < src1_nrows; ir1 += 2) { \ + const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); \ + const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); \ + float * restrict dst_row0 = (float *) (dst->data + ((cur_m_start + ir1+0) * dst_row_size)); \ + float * restrict dst_row1 = (float *) (dst->data + ((cur_m_start + ir1+1) * dst_row_size)); \ + \ + float * dst_ptr0 = &dst_row0[ct * 32]; \ + float * dst_ptr1 = &dst_row1[ct * 32]; \ + \ + const float * src2_ptr0 = NULL; \ + const float * src2_ptr1 = NULL; \ + if (src2) { \ + const float * restrict src2_row0 = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1+0) * src2_stride)); \ + const float * restrict src2_row1 = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1+1) * src2_stride)); \ + src2_ptr0 = &src2_row0[ct * 32]; \ + src2_ptr1 = &src2_row1[ct * 32]; \ + } \ + DOT_2X2(ne10, dst_ptr0, dst_ptr1, w_tile, src1_col0, src1_col1, valid_rows, src2_ptr0, src2_ptr1); \ + } \ + \ + for (; ir1 < src1_nrows; ++ir1) { \ + const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); \ + float * restrict dst_row = (float *) (dst->data + ((cur_m_start + ir1) * dst_row_size)); \ + float * dst_ptr = &dst_row[ct * 32]; \ + \ + const float * src2_ptr = NULL; \ + if (src2) { \ + const float * restrict src2_row = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1) * src2_stride)); \ + src2_ptr = &src2_row[ct * 32]; \ + } \ + DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, src2_ptr); \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ + \ + if (push_ct < ct_end) { \ + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), \ + aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + push_ct++; \ + } \ + } \ } -#define MATVEC_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X1) \ -static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ - htp_matmul_preamble; \ - \ - const uint32_t src0_nrows = ne01; \ - \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ - \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - \ - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ - const uint32_t n_prefetch = kparams->n_prefetch; \ - assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \ - \ - const size_t dst_row_size = nb1; \ - const size_t src1_row_size = nb11; \ - const size_t src1_stride = mmctx->vtcm_src1_stride; \ - \ - uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; \ - uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ - uint8_t * src1_data = mmctx->vtcm_src1; \ - \ - float * tmp = (float *) vtcm_dst_ptr; \ - \ - const uint8_t * restrict src0_row = (const uint8_t *) src0->data; \ - \ - const uint8_t * restrict src1_col = (const uint8_t *) src1_data; \ - float * restrict dst_col = (float *) dst->data; \ - \ - const uint32_t tile_size = TILE_SIZE; \ - const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \ - \ - uint32_t n_k_tiles_w = ne00 / 32; \ - uint32_t n_k_tiles_a = ne10 / 32; \ - uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ - uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ - \ - uint32_t ct_start = src0_start_row / 32; \ - uint32_t ct_end = (src0_end_row + 31) / 32; \ - \ - uint32_t push_ct = ct_start; \ - if (src0_start_row < src0_end_row) { \ - if (src2) { \ - float * vtcm_src2_ptr = (float *) mmctx->vtcm_src2 + src0_start_row; \ - const float * src2_ptr = (const float *) src2->data + src0_start_row; \ - int slice_size = (int)MIN(src0_end_row, ne0) - (int)src0_start_row; \ - if (slice_size > 0) { \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr, src2_ptr), \ - slice_size * sizeof(float), slice_size * sizeof(float), slice_size * sizeof(float), 1); \ - dma_queue_pop_nowait(dma_queue); \ - } \ - } \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ - src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - } \ - } \ - \ - hvx_mm_run_quant_task(mmctx, ith); \ - \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ - const uint8_t * w_tile = dma_queue_pop(dma_queue).dst; \ - \ - float * dst_ptr = &tmp[ct * 32 - src0_start_row]; \ - int valid_rows = (int)ne0 - (int)(ct * 32); \ - valid_rows = MIN(32, MAX(0, valid_rows)); \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, NULL); \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - \ - if (push_ct < ct_end) { \ - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile, src0_row + push_ct * tile_row_stride), \ - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - push_ct++; \ - } \ - } \ - \ - int copy_cnt = (int)MIN(src0_end_row, ne0) - (int)src0_start_row; \ - if (copy_cnt > 0) { \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct_end); \ - if (src2) { \ - hvx_add_f32_uaa((uint8_t *) &dst_col[src0_start_row], \ - (const uint8_t *) tmp, \ - (const uint8_t *) ((const float *) mmctx->vtcm_src2 + src0_start_row), \ - copy_cnt); \ - } else { \ - hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, copy_cnt); \ - } \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct_end); \ - } \ +#define MATVEC_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X1) \ +static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ + htp_matmul_preamble; \ + \ + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \ + \ + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \ + \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + \ + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ + const uint32_t n_prefetch = kparams->n_prefetch; \ + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \ + \ + const size_t dst_row_size = nb1; \ + const size_t src1_row_size = nb11; \ + const size_t src1_stride = mmctx->vtcm_src1_stride; \ + \ + uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; \ + uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ + uint8_t * src1_data = mmctx->vtcm_src1; \ + \ + float * tmp = (float *) vtcm_dst_ptr; \ + \ + const dma_addr_t src0_row = src0->data; \ + \ + const uint8_t * restrict src1_col = (const uint8_t *) src1_data; \ + float * restrict dst_col = (float *) dst->data; \ + \ + const uint32_t tile_size = TILE_SIZE; \ + const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \ + \ + uint32_t n_k_tiles_w = ne00 / 32; \ + uint32_t n_k_tiles_a = ne10 / 32; \ + uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ + uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ + \ + uint32_t ct_start = src0_start_row / 32; \ + uint32_t ct_end = (src0_end_row + 31) / 32; \ + \ + uint32_t push_ct = ct_start; \ + if (src0_start_row < src0_end_row) { \ + if (src2) { \ + float * vtcm_src2_ptr = (float *) mmctx->vtcm_src2 + src0_start_row; \ + const dma_addr_t src2_addr = src2->data + src0_start_row * sizeof(float); \ + int slice_size = (int)MIN(src0_end_row, ne0) - (int)src0_start_row; \ + if (slice_size > 0) { \ + dma_queue_push(dma_q, dma_make_data(vtcm_src2_ptr, src2_addr), \ + slice_size * sizeof(float), slice_size * sizeof(float), slice_size * sizeof(float), 1); \ + dma_queue_pop_nowait(dma_q); \ + } \ + } \ + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ + src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + } \ + } \ + \ + hvx_mm_run_quant_task(mmctx, ith); \ + \ + if (src0_start_row >= src0_end_row) { \ + return; \ + } \ + \ + for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; \ + \ + float * dst_ptr = &tmp[ct * 32 - src0_start_row]; \ + int valid_rows = (int)ne0 - (int)(ct * 32); \ + valid_rows = MIN(32, MAX(0, valid_rows)); \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ + DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, NULL); \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ + \ + if (push_ct < ct_end) { \ + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), \ + aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + push_ct++; \ + } \ + } \ + \ + int copy_cnt = (int)MIN(src0_end_row, ne0) - (int)src0_start_row; \ + if (copy_cnt > 0) { \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct_end); \ + if (src2) { \ + hvx_add_f32_uaa((uint8_t *) &dst_col[src0_start_row], \ + (const uint8_t *) tmp, \ + (const uint8_t *) ((const float *) mmctx->vtcm_src2 + src0_start_row), \ + copy_cnt); \ + } else { \ + hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, copy_cnt); \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct_end); \ + } \ } #define MATMUL_NX_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \ @@ -532,29 +479,38 @@ static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, v uint32_t n_k_tiles_a = ne10 / 32; \ uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ \ - dma_queue * dma_queue = octx->ctx->dma[ith]; \ - \ hvx_mm_run_quant_task(mmctx, ith); \ \ for (uint32_t widx = 0; widx < n_weights; widx++) { \ const struct htp_tensor * restrict src_w = octx->src[widx]; \ const struct htp_tensor * restrict dst = octx->dsts[widx]; \ if (!src_w || !dst) continue; \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ \ const uint32_t ne00 = src_w->ne[0]; \ const uint32_t ne01 = src_w->ne[1]; \ const size_t dst_row_size = dst->nb[1]; \ - const uint8_t * restrict src_w_row = (const uint8_t *) src_w->data; \ + const dma_addr_t src_w_row = src_w->data; \ \ uint32_t n_k_tiles_w = ne00 / 32; \ uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ \ - const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3]; \ - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); \ + uint32_t src0_start_row = 0; \ + uint32_t src0_end_row = ne01; \ + if (octx->ctx->mdev.count > 1) { \ + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); \ + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, \ + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); \ + src0_start_row = range.start; \ + src0_end_row = range.start + range.count; \ + } \ + \ + const uint32_t nrows = src0_end_row - src0_start_row; \ + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); \ src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); \ \ - const uint32_t start_row = src0_nrows_per_thread * ith; \ - const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_nrows); \ + const uint32_t start_row = src0_start_row + src0_nrows_per_thread * ith; \ + const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_end_row); \ if (start_row >= end_row) continue; \ \ uint32_t ct_start = start_row / 32; \ @@ -562,12 +518,12 @@ static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, v \ uint32_t push_ct = ct_start; \ for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_weight_ptr + d * tile_row_transfer_size_aligned, \ + dma_queue_push(dma_q, dma_make_data(vtcm_weight_ptr + d * tile_row_transfer_size_aligned, \ src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ } \ \ for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ - const uint8_t * w_tile = dma_queue_pop(dma_queue).dst; \ + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; \ int valid_rows = (int)ne01 - (int)(ct * 32); \ valid_rows = MIN(32, MAX(0, valid_rows)); \ \ @@ -594,7 +550,7 @@ static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, v htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ \ if (push_ct < ct_end) { \ - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile, src_w_row + push_ct * tile_row_stride), \ + dma_queue_push(dma_q, dma_make_data(w_tile, src_w_row + push_ct * tile_row_stride), \ aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ push_ct++; \ } \ @@ -605,54 +561,61 @@ static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, v MATMUL_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1) MATMUL_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1) MATMUL_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1) +MATMUL_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x2, tiled_vec_dot_q6_k_32x1) MATMUL_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1) MATMUL_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) -MATMUL_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x2, flat_vec_dot_q4_0_32x1) -MATMUL_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x2, flat_vec_dot_q4_1_32x1) -MATMUL_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x2, flat_vec_dot_q8_0_32x1) -MATMUL_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x2, flat_vec_dot_iq4nl_32x1) -MATMUL_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot_mxfp4_32x1) - -#define QUANTIZE_IMPL(name, log_name, kernel_fn, dst_row_size_expr) \ -static void name(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_mm_context * mmctx = data; \ - struct htp_ops_context * octx = mmctx->octx; \ - const struct htp_tensor * src = mmctx->act; \ - const uint32_t ne0 = src->ne[0]; \ - const uint32_t ne1 = src->ne[1]; \ - const uint32_t ne2 = src->ne[2]; \ - const uint32_t ne3 = src->ne[3]; \ - const uint32_t nrows = ne1 * ne2 * ne3; \ - const uint32_t nrows_per_thread = mmctx->n_quant_rows_per_thread; \ - \ - const uint32_t ir_first = nrows_per_thread * ith; \ - if (ir_first >= nrows) { \ - return; \ - } \ - \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ - \ - uint8_t * restrict dst = mmctx->vtcm_src1; \ - const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); \ - const size_t src_row_size = src->nb[1]; \ - const size_t dst_row_size = (dst_row_size_expr); \ - const uint8_t * restrict src_data = (const uint8_t *) src->data + (src_row_size * ir_first); \ - uint8_t * restrict dst_data = (uint8_t *) dst + (dst_row_size * ir_first); \ - uint8_t * restrict tmp_data = (uint8_t *) mmctx->vtcm_dst + (mmctx->vtcm_dst_size_per_thread * ith); \ - kernel_fn(src_data, dst_data, tmp_data, ne0, ir_last - ir_first, src_row_size, dst_row_size); \ - \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ +#define QUANTIZE_IMPL(name, log_name, kernel_fn, dst_row_size_expr) \ +static void name(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_mm_context * mmctx = data; \ + struct htp_ops_context * octx = mmctx->octx; \ + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ + const struct htp_tensor * src = mmctx->act; \ + const uint32_t ne0 = src->ne[0]; \ + const uint32_t nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->src1_nrows; \ + const uint32_t nrows_per_thread = mmctx->n_quant_rows_per_thread; \ + \ + const uint32_t ir_first = nrows_per_thread * ith; \ + if (ir_first >= nrows) { \ + return; \ + } \ + \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ + \ + uint8_t * restrict dst = mmctx->vtcm_src1; \ + const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); \ + const size_t src_row_size = src->nb[1]; \ + const size_t dst_row_size = (dst_row_size_expr); \ + uint8_t * restrict tmp_data = (uint8_t *) mmctx->vtcm_dst + (mmctx->vtcm_dst_size_per_thread * ith); \ + \ + const bool is_contiguous = (src->nb[2] == src->ne[1] * src->nb[1]) && (src->nb[3] == src->ne[2] * src->nb[2]); \ + if (is_contiguous) { \ + const uint8_t * restrict src_data = (const uint8_t *) src->data + (src_row_size * (mmctx->cur_m_start + ir_first)); \ + uint8_t * restrict dst_data = (uint8_t *) dst + (dst_row_size * ir_first); \ + kernel_fn(src_data, dst_data, tmp_data, ne0, ir_last - ir_first, src_row_size, dst_row_size); \ + } else { \ + const uint32_t ne12_ne1 = src->ne[2] * src->ne[1]; \ + for (uint32_t ir = ir_first; ir < ir_last; ++ir) { \ + const uint32_t ir1 = mmctx->cur_m_start + ir; \ + const uint32_t i13 = fastdiv(ir1, &kparams->div_ne12_ne1); \ + const uint32_t rem = ir1 - i13 * ne12_ne1; \ + const uint32_t i12 = fastdiv(rem, &kparams->div_ne1); \ + const uint32_t i11 = rem - i12 * src->ne[1]; \ + const uint8_t * restrict row_src = (const uint8_t *) src->data + ((size_t) i11 * src->nb[1] + (size_t) i12 * src->nb[2] + (size_t) i13 * src->nb[3]); \ + uint8_t * restrict row_dst = dst + (dst_row_size * ir); \ + kernel_fn(row_src, row_dst, tmp_data, ne0, 1, src_row_size, dst_row_size); \ + } \ + } \ + \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ } QUANTIZE_IMPL(quantize_f32_q8_0_tiled, "quantize-f32-q8_0_tiled", quantize_f32_q8_0_tiled_kernel, htp_mm_q8_0_tiled_row_size(ne0)) QUANTIZE_IMPL(quantize_f32_q8_1_tiled, "quantize-f32-q8_1_tiled", quantize_f32_q8_1_tiled_kernel, htp_mm_q8_1_tiled_row_size(ne0)) -QUANTIZE_IMPL(quantize_f32_q8_0_flat, "quantize-f32-q8_0_flat", quantize_f32_q8_0_flat_kernel, htp_mm_q8_0_flat_row_size(ne0)) -QUANTIZE_IMPL(quantize_f32_q8_1_flat, "quantize-f32-q8_1_flat", quantize_f32_q8_1_flat_kernel, htp_mm_q8_1_flat_row_size(ne0)) -QUANTIZE_IMPL(quantize_f32_f32_flat, "quantize-f32-f32", quantize_f32_f32_flat_kernel, mmctx->vtcm_src1_stride) -QUANTIZE_IMPL(quantize_f32_f16_flat, "quantize-f32-f16", quantize_f32_f16_flat_kernel, mmctx->vtcm_src1_stride) -QUANTIZE_IMPL(quantize_f16_f16_flat, "quantize-f16-f16", quantize_f16_f16_flat_kernel, mmctx->vtcm_src1_stride) +QUANTIZE_IMPL(quantize_f32_f32, "quantize-f32-f32", quantize_f32_f32_kernel, mmctx->vtcm_src1_stride) +QUANTIZE_IMPL(quantize_f32_f16, "quantize-f32-f16", quantize_f32_f16_kernel, mmctx->vtcm_src1_stride) +QUANTIZE_IMPL(quantize_f16_f16, "quantize-f16-f16", quantize_f16_f16_kernel, mmctx->vtcm_src1_stride) static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = data; @@ -705,27 +668,150 @@ static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, vo MATVEC_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x1) MATVEC_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x1) MATVEC_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x1) +MATVEC_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x1) MATVEC_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x1) MATVEC_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x1) -MATVEC_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x1) -MATVEC_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x1) -MATVEC_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x1) -MATVEC_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x1) -MATVEC_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x1) - - MATMUL_NX_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1) MATMUL_NX_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1) MATMUL_NX_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1) MATMUL_NX_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1) MATMUL_NX_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) -MATMUL_NX_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x2, flat_vec_dot_q4_0_32x1) -MATMUL_NX_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x2, flat_vec_dot_q4_1_32x1) -MATMUL_NX_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x2, flat_vec_dot_q8_0_32x1) -MATMUL_NX_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x2, flat_vec_dot_iq4nl_32x1) -MATMUL_NX_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot_mxfp4_32x1) +#define MATMUL_4D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \ +static void hvx_mm_4d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ + htp_matmul_preamble; \ + \ + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \ + const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); \ + const uint32_t cur_m_start = mmctx->cur_m_start; \ + \ + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \ + \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + \ + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ + const uint32_t n_prefetch = kparams->n_prefetch; \ + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \ + \ + const size_t dst_row_size = nb1; \ + const size_t src1_stride = mmctx->vtcm_src1_stride; \ + \ + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ + uint8_t * restrict src1_data = mmctx->vtcm_src1; \ + \ + const uint32_t tile_size = TILE_SIZE; \ + const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \ + \ + const uint32_t n_k_tiles_w = ne00 / 32; \ + const uint32_t n_k_tiles_a = ne10 / 32; \ + const uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \ + const uint32_t src0_slice_stride = ((ne01 + 31) / 32) * tile_row_stride; \ + \ + const uint32_t ct_start = src0_start_row / 32; \ + const uint32_t ct_end = (src0_end_row + 31) / 32; \ + \ + hvx_mm_run_quant_task(mmctx, ith); \ + \ + if (src0_start_row >= src0_end_row || cur_m_rows == 0) { \ + return; \ + } \ + \ + const uint32_t total_batches = ne12 * ne13; \ + const uint32_t b_start = fastdiv(cur_m_start, &kparams->div_ne1); \ + uint32_t b_end = fastdiv(cur_m_start + cur_m_rows + ne11 - 1, &kparams->div_ne1); \ + b_end = MIN(b_end, total_batches); \ + \ + uint32_t b_grp_start = b_start; \ + while (b_grp_start < b_end) { \ + const uint32_t b3 = fastdiv(b_grp_start, &kparams->div_ne12); \ + const uint32_t b2 = b_grp_start - b3 * ne12; \ + const uint32_t i02 = fastdiv(b2, &kparams->div_r2); \ + const uint32_t i03 = fastdiv(b3, &kparams->div_r3); \ + \ + uint32_t b_grp_end = b_grp_start + 1; \ + while (b_grp_end < b_end) { \ + const uint32_t cur_b3 = fastdiv(b_grp_end, &kparams->div_ne12); \ + const uint32_t cur_b2 = b_grp_end - cur_b3 * ne12; \ + const uint32_t cur_i02 = fastdiv(cur_b2, &kparams->div_r2); \ + const uint32_t cur_i03 = fastdiv(cur_b3, &kparams->div_r3); \ + if (cur_i02 != i02 || cur_i03 != i03) { \ + break; \ + } \ + b_grp_end++; \ + } \ + \ + const uint32_t slice_idx = i03 * ne02 + i02; \ + const dma_addr_t src0_slice = src0->data + (size_t) slice_idx * src0_slice_stride; \ + \ + uint32_t push_ct = ct_start; \ + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ + src0_slice + (size_t) push_ct * tile_row_stride), \ + aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + } \ + \ + for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; \ + \ + int valid_rows = (int)ne0 - (int)(ct * 32); \ + valid_rows = MIN(32, MAX(0, valid_rows)); \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ + for (uint32_t b = b_grp_start; b < b_grp_end; b++) { \ + const uint32_t b_m_start = b * ne11; \ + const uint32_t m_first = MAX(cur_m_start, b_m_start); \ + const uint32_t m_last = MIN(cur_m_start + cur_m_rows, b_m_start + ne11); \ + if (m_first >= m_last) continue; \ + \ + const uint32_t cur_b3 = fastdiv(b, &kparams->div_ne12); \ + const uint32_t cur_b2 = b - cur_b3 * ne12; \ + uint8_t * dst_batch_base = (uint8_t *) dst->data + (size_t) cur_b2 * nb2 + (size_t) cur_b3 * nb3; \ + \ + const uint32_t chunk_m_offset = m_first - cur_m_start; \ + const uint32_t dst_m_offset = m_first - b_m_start; \ + const uint32_t batch_nrows = m_last - m_first; \ + \ + uint32_t ir1 = 0; \ + for (; ir1 + 1 < batch_nrows; ir1 += 2) { \ + const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (chunk_m_offset + ir1 + 0) * src1_stride); \ + const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (chunk_m_offset + ir1 + 1) * src1_stride); \ + float * restrict dst_row0 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 0) * dst_row_size); \ + float * restrict dst_row1 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 1) * dst_row_size); \ + float * dst_ptr0 = &dst_row0[ct * 32]; \ + float * dst_ptr1 = &dst_row1[ct * 32]; \ + DOT_2X2(ne10, dst_ptr0, dst_ptr1, w_tile, src1_col0, src1_col1, valid_rows, NULL, NULL); \ + } \ + for (; ir1 < batch_nrows; ++ir1) { \ + const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (chunk_m_offset + ir1) * src1_stride); \ + float * restrict dst_row = (float *) (dst_batch_base + (dst_m_offset + ir1) * dst_row_size); \ + float * dst_ptr = &dst_row[ct * 32]; \ + DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, NULL); \ + } \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ + \ + if (push_ct < ct_end) { \ + dma_queue_push(dma_q, dma_make_data(w_tile, src0_slice + (size_t) push_ct * tile_row_stride), \ + aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + push_ct++; \ + } \ + } \ + b_grp_start = b_grp_end; \ + } \ + if (src2) { \ + hvx_tensor_add_f32_grid(dst, src2, cur_m_start, cur_m_start + cur_m_rows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); \ + } \ +} + +MATMUL_4D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1) +MATMUL_4D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1) +MATMUL_4D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1) +MATMUL_4D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x2, tiled_vec_dot_q6_k_32x1) +MATMUL_4D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1) +MATMUL_4D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; @@ -735,11 +821,12 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); const uint32_t prefetch_mask = n_prefetch - 1; - const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows - const uint32_t src1_nrows = ne11 * ne12 * ne13; // src1 rows + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows + const uint32_t src1_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->src1_nrows; // src1 rows + const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); struct htp_thread_trace * tr = &octx->ctx->trace[ith]; @@ -756,7 +843,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; uint8_t * restrict src1_data = mmctx->vtcm_src1; - const uint8_t * restrict src0_row = (const uint8_t *) src0->data; + const dma_addr_t src0_row = src0->data; // Prefill vtcm with src0 rows if (src0_start_row < src0_end_row) { @@ -765,7 +852,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { if (is0 >= (int)n_prefetch) { break; } - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), src0_stride, src0_row_size, src0_row_size, 2); } } @@ -778,23 +865,23 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { // Process src0 rows for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - // Process src1 columns in pairs (2×2 tiling) + // Process src1 columns in pairs (2x2 tiling) uint32_t ir1 = 0; for (; ir1 + 1 < src1_nrows; ir1 += 2) { const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); - float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size)); - float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size)); + float * restrict dst_row0 = (float *) (dst->data + ((cur_m_start + ir1+0) * dst_row_size)); + float * restrict dst_row1 = (float *) (dst->data + ((cur_m_start + ir1+1) * dst_row_size)); mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1); } - // Handle remaining src1 rows (fallback to 2×1) + // Handle remaining src1 rows (fallback to 2x1) for (; ir1 < src1_nrows; ++ir1) { const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); - float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); + float * restrict dst_row = (float *) (dst->data + ((cur_m_start + ir1) * dst_row_size)); mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, src1_col); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); @@ -803,7 +890,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { const int pr0 = (ir0 + n_prefetch); const int is0 = (pr0 - src0_start_row) & prefetch_mask; if (pr0 < src0_end_row_x2) { - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), src0_stride, src0_row_size, src0_row_size, 2); } } @@ -812,31 +899,31 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { if (src0_end_row != src0_end_row_x2) { uint32_t ir0 = src0_end_row_x2; const int is0 = (ir0 - src0_start_row) & prefetch_mask; - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), src0_stride, src0_row_size, src0_row_size, 1); - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); #pragma unroll(2) for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) { const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); - float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); + float * restrict dst_row = (float *) (dst->data + ((cur_m_start + ir1) * dst_row_size)); mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); } if (src2) { - hvx_tensor_add_f32_grid(dst, src2, 0, src1_nrows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); + hvx_tensor_add_f32_grid(dst, src2, cur_m_start, cur_m_start + src1_nrows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); } } static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; - const uint32_t src0_nrows = ne01; + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); struct htp_thread_trace * tr = &octx->ctx->trace[ith]; @@ -854,7 +941,7 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { float * tmp = (float *) vtcm_dst_ptr; - const uint8_t * restrict src0_row = (const uint8_t *) src0->data; + const dma_addr_t src0_row = src0->data; const uint8_t * restrict src1_col = (const uint8_t *) src1_data; float * restrict dst_col = (float *) dst->data; @@ -869,12 +956,12 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { if (src0_start_row < src0_end_row) { if (src2) { float * vtcm_src2_ptr = (float *) mmctx->vtcm_src2 + src0_start_row; - const float * src2_ptr = (const float *) src2->data + src0_start_row; + const dma_addr_t src2_addr = src2->data + src0_start_row * sizeof(float); int slice_size = (int)src0_end_row - (int)src0_start_row; if (slice_size > 0) { - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr, src2_ptr), + dma_queue_push(dma_q, dma_make_data(vtcm_src2_ptr, src2_addr), slice_size * sizeof(float), slice_size * sizeof(float), slice_size * sizeof(float), 1); - dma_queue_pop_nowait(dma_queue); + dma_queue_pop_nowait(dma_q); } } for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { @@ -882,7 +969,7 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { if (is0 >= n_prefetch) { break; } - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), src0_stride, src0_row_size, src0_row_size, 2); } } @@ -895,7 +982,7 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { // Process src0 rows for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); mmctx->vec_dot_2x1(ne00, &tmp[ir0 - src0_start_row], ss0, ss0 + src0_stride, src1_col); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); @@ -904,7 +991,7 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t pr0 = (ir0 + n_prefetch); const uint32_t is0 = (pr0 - src0_start_row) & prefetch_mask; if (pr0 < src0_end_row_x2) { - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), src0_stride, src0_row_size, src0_row_size, 2); } } @@ -913,9 +1000,9 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { if (src0_end_row != src0_end_row_x2) { const uint32_t ir0 = src0_end_row_x2; const uint32_t is0 = (ir0 - src0_start_row) & prefetch_mask; - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), src0_stride, src0_row_size, src0_row_size, 1); - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); mmctx->vec_dot_1x1(ne00, &tmp[ir0 - src0_start_row], ss0, src1_col); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); @@ -936,6 +1023,154 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { } } +static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; + + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); + const uint32_t prefetch_mask = n_prefetch - 1; + + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; + const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); + const uint32_t cur_m_start = mmctx->cur_m_start; + + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + const size_t dst_row_size = nb1; + const size_t src0_row_size = nb01; + const size_t src0_stride = mmctx->vtcm_src0_stride; + const size_t src1_stride = mmctx->vtcm_src1_stride; + + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict src1_data = mmctx->vtcm_src1; + + hvx_mm_run_quant_task(mmctx, ith); + + if (src0_start_row >= src0_end_row || cur_m_rows == 0) { + return; + } + + const uint32_t total_batches = ne12 * ne13; + const uint32_t b_start = fastdiv(cur_m_start, &kparams->div_ne1); + uint32_t b_end = fastdiv(cur_m_start + cur_m_rows + ne11 - 1, &kparams->div_ne1); + b_end = MIN(b_end, total_batches); + + uint32_t b_grp_start = b_start; + while (b_grp_start < b_end) { + const uint32_t b3 = fastdiv(b_grp_start, &kparams->div_ne12); + const uint32_t b2 = b_grp_start - b3 * ne12; + const uint32_t i02 = fastdiv(b2, &kparams->div_r2); + const uint32_t i03 = fastdiv(b3, &kparams->div_r3); + + uint32_t b_grp_end = b_grp_start + 1; + while (b_grp_end < b_end) { + const uint32_t cur_b3 = fastdiv(b_grp_end, &kparams->div_ne12); + const uint32_t cur_b2 = b_grp_end - cur_b3 * ne12; + const uint32_t cur_i02 = fastdiv(cur_b2, &kparams->div_r2); + const uint32_t cur_i03 = fastdiv(cur_b3, &kparams->div_r3); + if (cur_i02 != i02 || cur_i03 != i03) { + break; + } + b_grp_end++; + } + + const dma_addr_t src0_row = src0->data + ((size_t) i02 * nb02 + (size_t) i03 * nb03); + + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const int is0 = (ir0 - src0_start_row); + if (is0 >= (int)n_prefetch) { + break; + } + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + (size_t) ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); + } + + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + for (uint32_t b = b_grp_start; b < b_grp_end; b++) { + const uint32_t b_m_start = b * ne11; + const uint32_t m_first = MAX(cur_m_start, b_m_start); + const uint32_t m_last = MIN(cur_m_start + cur_m_rows, b_m_start + ne11); + if (m_first >= m_last) continue; + + const uint32_t cur_b3 = fastdiv(b, &kparams->div_ne12); + const uint32_t cur_b2 = b - cur_b3 * ne12; + uint8_t * dst_batch_base = (uint8_t *) dst->data + (size_t) cur_b2 * nb2 + (size_t) cur_b3 * nb3; + + const uint32_t chunk_m_offset = m_first - cur_m_start; + const uint32_t dst_m_offset = m_first - b_m_start; + const uint32_t batch_nrows = m_last - m_first; + + uint32_t ir1 = 0; + for (; ir1 + 1 < batch_nrows; ir1 += 2) { + const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (chunk_m_offset + ir1 + 0) * src1_stride); + const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (chunk_m_offset + ir1 + 1) * src1_stride); + float * restrict dst_row0 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 0) * dst_row_size); + float * restrict dst_row1 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 1) * dst_row_size); + mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1); + } + for (; ir1 < batch_nrows; ++ir1) { + const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (chunk_m_offset + ir1) * src1_stride); + float * restrict dst_row = (float *) (dst_batch_base + (dst_m_offset + ir1) * dst_row_size); + mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, src1_col); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + + const int pr0 = (ir0 + n_prefetch); + const int is0 = (pr0 - src0_start_row) & prefetch_mask; + if (pr0 < src0_end_row_x2) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + (size_t) pr0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); + } + } + + if (src0_end_row != src0_end_row_x2) { + uint32_t ir0 = src0_end_row_x2; + const int is0 = (ir0 - src0_start_row) & prefetch_mask; + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + (size_t) ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 1); + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + for (uint32_t b = b_grp_start; b < b_grp_end; b++) { + const uint32_t b_m_start = b * ne11; + const uint32_t m_first = MAX(cur_m_start, b_m_start); + const uint32_t m_last = MIN(cur_m_start + cur_m_rows, b_m_start + ne11); + if (m_first >= m_last) continue; + + const uint32_t cur_b3 = fastdiv(b, &kparams->div_ne12); + const uint32_t cur_b2 = b - cur_b3 * ne12; + uint8_t * dst_batch_base = (uint8_t *) dst->data + (size_t) cur_b2 * nb2 + (size_t) cur_b3 * nb3; + + const uint32_t chunk_m_offset = m_first - cur_m_start; + const uint32_t dst_m_offset = m_first - b_m_start; + const uint32_t batch_nrows = m_last - m_first; + + for (uint32_t ir1 = 0; ir1 < batch_nrows; ++ir1) { + const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (chunk_m_offset + ir1) * src1_stride); + float * restrict dst_row = (float *) (dst_batch_base + (dst_m_offset + ir1) * dst_row_size); + mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + } + + b_grp_start = b_grp_end; + } + + if (src2) { + hvx_tensor_add_f32_grid(dst, src2, cur_m_start, cur_m_start + cur_m_rows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); + } +} + #define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) * mmctx->mapping_stride + (i1)] static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { @@ -943,13 +1178,10 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict ids = octx->src[2]; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t src0_nrows = ne01; // src0 rows per expert + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert const uint32_t src1_nrows = ne11; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); hvx_mm_run_quant_task(mmctx, ith); @@ -984,7 +1216,7 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { continue; } - const uint8_t * src0_row = (const uint8_t *) src0->data + cur_a * nb02; + const dma_addr_t src0_row = src0->data + cur_a * nb02; const uint32_t tile_size = htp_mm_get_weight_tile_size(src0->type); const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(src0->type); @@ -998,12 +1230,12 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { - const uint8_t * w_tile = dma_queue_pop(dma_queue).dst; + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; int valid_rows = (int)ne01 - (int)(ct * 32); valid_rows = MIN(32, MAX(0, valid_rows)); @@ -1023,7 +1255,7 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); if (push_ct < ct_end) { - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile, src0_row + push_ct * tile_row_stride), + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); push_ct++; } @@ -1036,9 +1268,9 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict ids = octx->src[2]; - const uint32_t src0_nrows = ne01; // src0 rows per expert - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); hvx_mm_run_quant_task(mmctx, ith); @@ -1071,7 +1303,7 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { } assert(eid < (int32_t) n_ids); - const uint8_t * restrict src0_row = (const uint8_t *) src0->data + eid * nb02; + const dma_addr_t src0_row = src0->data + eid * nb02; const uint8_t * restrict src1_col = (const uint8_t *) src1_data; float * restrict dst_row = (float *) (dst->data + ie1 * nb1); @@ -1087,12 +1319,12 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { - const uint8_t * w_tile = dma_queue_pop(dma_queue).dst; + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; int valid_rows = (int)ne01 - (int)(ct * 32); valid_rows = MIN(32, MAX(0, valid_rows)); @@ -1102,7 +1334,7 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); if (push_ct < ct_end) { - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile, src0_row + push_ct * tile_row_stride), + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); push_ct++; } @@ -1113,7 +1345,6 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = (struct htp_mm_context *) data; struct htp_ops_context * octx = mmctx->octx; - dma_queue * dma_queue = octx->ctx->dma[ith]; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_weights = kparams->n_weights; const struct htp_tensor * restrict src0 = octx->src[0]; @@ -1142,16 +1373,27 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict src_w = octx->src[p]; const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; + dma_queue * dma_q = octx->ctx->dma[ith]; + + const uint32_t ne01 = src_w->ne[1]; + uint32_t start_row = 0; + uint32_t end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + start_row = range.start; + end_row = range.start + range.count; + } - const uint32_t src0_nrows = src_w->ne[1]; - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); + const uint32_t nrows = end_row - start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); if (src0_start_row >= src0_end_row) continue; - const uint8_t * restrict src0_row = (const uint8_t *) src_w->data + eid * src_w->nb[2]; + const dma_addr_t src0_row = src_w->data + eid * src_w->nb[2]; const uint8_t * restrict src1_col = (const uint8_t *) src1_data; float * restrict dst_row = (float *) (dst->data + ie1 * dst->nb[1]); @@ -1167,12 +1409,12 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { - const uint8_t * w_tile = dma_queue_pop(dma_queue).dst; + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; int valid_rows = (int)src_w->ne[1] - (int)(ct * 32); valid_rows = MIN(32, MAX(0, valid_rows)); @@ -1182,7 +1424,7 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); if (push_ct < ct_end) { - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile, src0_row + push_ct * tile_row_stride), + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); push_ct++; } @@ -1194,7 +1436,6 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = (struct htp_mm_context *) data; struct htp_ops_context * octx = mmctx->octx; - dma_queue * dma_queue = octx->ctx->dma[ith]; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_weights = kparams->n_weights; const struct htp_tensor * restrict src0 = octx->src[0]; @@ -1226,16 +1467,27 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict src_w = octx->src[p]; const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; + dma_queue * dma_q = octx->ctx->dma[ith]; + + const uint32_t ne01 = src_w->ne[1]; + uint32_t start_row = 0; + uint32_t end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + start_row = range.start; + end_row = range.start + range.count; + } - const uint32_t src0_nrows = src_w->ne[1]; - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); + const uint32_t nrows = end_row - start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); if (src0_start_row >= src0_end_row) continue; - const uint8_t * src0_row = (const uint8_t *) src_w->data + cur_a * src_w->nb[2]; + const dma_addr_t src0_row = src_w->data + cur_a * src_w->nb[2]; const uint32_t tile_size = htp_mm_get_weight_tile_size(src_w->type); const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(src_w->type); @@ -1249,12 +1501,12 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { - const uint8_t * w_tile = dma_queue_pop(dma_queue).dst; + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; int valid_rows = (int)src_w->ne[1] - (int)(ct * 32); valid_rows = MIN(32, MAX(0, valid_rows)); @@ -1274,7 +1526,7 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); if (push_ct < ct_end) { - dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile, src0_row + push_ct * tile_row_stride), + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); push_ct++; } @@ -1290,6 +1542,7 @@ static int hvx_mm_init_vec_dot(struct htp_mm_context * mmctx, enum htp_data_type mmctx->vec_dot_32x1 = tiled_vec_dot_q4_0_32x1; return 0; case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: mmctx->type = "q4_1_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_q4_1_32x1; return 0; @@ -1297,6 +1550,10 @@ static int hvx_mm_init_vec_dot(struct htp_mm_context * mmctx, enum htp_data_type mmctx->type = "q8_0_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_q8_0_32x1; return 0; + case HTP_TYPE_Q6_K: + mmctx->type = "q6_k_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q6_k_32x1; + return 0; case HTP_TYPE_IQ4_NL: mmctx->type = "iq4nl_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_iq4nl_32x1; @@ -1323,15 +1580,35 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - const uint32_t src0_nrows = ne01 * ne02 * ne03; + const uint32_t src0_nrows = ne01; const uint32_t src1_nrows = ne11 * ne12 * ne13; + mmctx->src1_nrows = src1_nrows; + + uint32_t src0_row_start = 0; + uint32_t src0_row_end = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + src0_row_start = range.start; + src0_row_end = range.start + range.count; + } + + if (src0_row_start >= src0_row_end) { + return HTP_STATUS_OK; + } + + const uint32_t nrows = src0_row_end - src0_row_start; + mmctx->src0_row_start = src0_row_start; + mmctx->src0_row_end = src0_row_end; bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL || - src0->type == HTP_TYPE_MXFP4); + src0->type == HTP_TYPE_MXFP4 || src0->type == HTP_TYPE_Q6_K || + src0->type == HTP_TYPE_Q4_K); // Compute src0_nrows_per_thread - mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div); + mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); if (is_repacked) { mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); } else { @@ -1348,12 +1625,30 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { worker_callback_t quant_task_func; worker_callback_t matmul_job_func; uint32_t n_quant_tasks = 1; - if (src1_nrows > 1) { + const bool is_batched = (ne12 > 1 || ne13 > 1 || ne02 > 1 || ne03 > 1); + if (is_batched) { + if (is_repacked) { + switch (src0->type) { + case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_4d_repacked_q4_0; break; + case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_4d_repacked_q4_1; break; + case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_4d_repacked_q8_0; break; + case HTP_TYPE_Q6_K: matmul_job_func = hvx_mm_4d_repacked_q6_k; break; + case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_4d_repacked_iq4nl; break; + case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_4d_repacked_mxfp4; break; + default: return HTP_STATUS_NO_SUPPORT; + } + } else { + matmul_job_func = hvx_mm_4d; + } + } else if (src1_nrows > 1) { if (is_repacked) { switch (src0->type) { case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_2d_repacked_q4_0; break; - case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_2d_repacked_q4_1; break; + case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_2d_repacked_q4_1; break; case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_2d_repacked_q8_0; break; + case HTP_TYPE_Q6_K: matmul_job_func = hvx_mm_2d_repacked_q6_k; break; case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_2d_repacked_iq4nl; break; case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_2d_repacked_mxfp4; break; default: return HTP_STATUS_NO_SUPPORT; @@ -1365,8 +1660,10 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { if (is_repacked) { switch (src0->type) { case HTP_TYPE_Q4_0: matmul_job_func = hvx_mv_2d_repacked_q4_0; break; - case HTP_TYPE_Q4_1: matmul_job_func = hvx_mv_2d_repacked_q4_1; break; + case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: matmul_job_func = hvx_mv_2d_repacked_q4_1; break; case HTP_TYPE_Q8_0: matmul_job_func = hvx_mv_2d_repacked_q8_0; break; + case HTP_TYPE_Q6_K: matmul_job_func = hvx_mv_2d_repacked_q6_k; break; case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mv_2d_repacked_iq4nl; break; case HTP_TYPE_MXFP4: matmul_job_func = hvx_mv_2d_repacked_mxfp4; break; default: return HTP_STATUS_NO_SUPPORT; @@ -1380,7 +1677,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { switch (kparams->kernel_type) { case HTP_MM_KERNEL_HVX_F16_F16_VTCM: - quant_task_func = (src1->type == HTP_TYPE_F32) ? quantize_f32_f16_flat : quantize_f16_f16_flat; + quant_task_func = (src1->type == HTP_TYPE_F32) ? quantize_f32_f16 : quantize_f16_f16; mmctx->type = "f16-f16"; mmctx->vec_dot_1x1 = vec_dot_f16_f16_aa_1x1; mmctx->vec_dot_2x1 = vec_dot_f16_f16_aa_2x1; @@ -1388,34 +1685,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { src1_row_size = hex_round_up(ne10 * 2, 128); break; - case HTP_MM_KERNEL_HVX_F16_F32_DDR: - mmctx->type = "f16-f32"; - mmctx->vec_dot_1x1 = vec_dot_f16_f32_uu_1x1; - matmul_job_func = hvx_mm_4d; - mmctx->mm_div_ne12_ne1 = kparams->div_ne12_ne1; - mmctx->mm_div_ne1 = kparams->div_ne1; - mmctx->mm_div_r2 = kparams->div_r2; - mmctx->mm_div_r3 = kparams->div_r3; - need_quant = false; - quant_task_func = NULL; - src1_row_size = nb11; - break; - - case HTP_MM_KERNEL_HVX_F16_F16_DDR: - mmctx->type = "f16-f16"; - mmctx->vec_dot_1x1 = vec_dot_f16_f16_uu_1x1; - matmul_job_func = hvx_mm_4d; - mmctx->mm_div_ne12_ne1 = kparams->div_ne12_ne1; - mmctx->mm_div_ne1 = kparams->div_ne1; - mmctx->mm_div_r2 = kparams->div_r2; - mmctx->mm_div_r3 = kparams->div_r3; - src1_row_size = nb11; - need_quant = false; - quant_task_func = NULL; - break; - case HTP_MM_KERNEL_HVX_F32_F32_VTCM: - quant_task_func = quantize_f32_f32_flat; + quant_task_func = quantize_f32_f32; mmctx->type = "f32-f32"; mmctx->vec_dot_1x1 = vec_dot_f32_f32_aa_1x1; mmctx->vec_dot_2x1 = vec_dot_f32_f32_aa_2x1; @@ -1423,46 +1694,6 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { src1_row_size = hex_round_up(ne10 * 4, 128); break; - case HTP_MM_KERNEL_HVX_F32_F32_DDR: - quant_task_func = NULL; - mmctx->type = "f32-f32"; - mmctx->vec_dot_1x1 = vec_dot_f32_f32_uu_1x1; - mmctx->mm_div_ne12_ne1 = kparams->div_ne12_ne1; - mmctx->mm_div_ne1 = kparams->div_ne1; - mmctx->mm_div_r2 = kparams->div_r2; - mmctx->mm_div_r3 = kparams->div_r3; - src1_row_size = nb11; - need_quant = false; - matmul_job_func = hvx_mm_4d; - break; - - case HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT: { - n_quant_tasks = MIN(src1_nrows, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat; - src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - - if (src1_nrows > 1) { - switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_2d_repacked_q4_0_flat; break; - case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_2d_repacked_q4_1_flat; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_2d_repacked_q8_0_flat; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_2d_repacked_iq4nl_flat; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_2d_repacked_mxfp4_flat; break; - default: return HTP_STATUS_NO_SUPPORT; - } - } else { - switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mv_2d_repacked_q4_0_flat; break; - case HTP_TYPE_Q4_1: matmul_job_func = hvx_mv_2d_repacked_q4_1_flat; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mv_2d_repacked_q8_0_flat; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mv_2d_repacked_iq4nl_flat; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mv_2d_repacked_mxfp4_flat; break; - default: return HTP_STATUS_NO_SUPPORT; - } - } - break; - } - case HTP_MM_KERNEL_HVX_QUANT_BLOCK: case HTP_MM_KERNEL_HVX_QUANT_ROW: default: @@ -1474,9 +1705,9 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const uint32_t nb = (ne10 + qk - 1) / qk; const uint32_t total_nb = src1_nrows * nb; - if (src1_nrows < octx->n_threads) { + if (src1_nrows < octx->n_threads && !is_batched) { n_quant_tasks = MIN(total_nb, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; + quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; @@ -1487,14 +1718,18 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { } } else { n_quant_tasks = MIN(src1_nrows, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; + quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; } - src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); break; } + const uint32_t m_chunk = (kparams->m_chunk > 0 && (uint32_t) kparams->m_chunk < src1_nrows) + ? (uint32_t) kparams->m_chunk : src1_nrows; + const uint32_t m_layout_rows = m_chunk; + struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads, + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, m_layout_rows, octx->n_threads, dst_row_size, src0_row_size, src1_row_size, src2 ? src2->nb[1] : 0, kparams->n_prefetch, false, false); if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || @@ -1503,13 +1738,13 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { mmctx->vtcm_src1_size_per_thread = L.src1_bytes; } else { - mmctx->vtcm_src1_size_per_thread = fastdiv(L.src1_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src1_size_per_thread = fastdiv(L.src1_bytes, &octx->n_threads_div); } - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; FARF(HIGH, "matmul-%s : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu (%zu)\n", mmctx->type, L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size); @@ -1537,19 +1772,46 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->vtcm_src0_stride = src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - if (need_quant) { - mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; - mmctx->quant_task_func = quant_task_func; - mmctx->n_quant_tasks = n_quant_tasks; - atomic_init(&mmctx->quant_barrier, n_quant_tasks); + if (kparams->m_chunk > 0 && (uint32_t) kparams->m_chunk < src1_nrows) { + atomic_init(&mmctx->quant_barrier, 0); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + + for (uint32_t m_start = 0; m_start < src1_nrows; m_start += m_chunk) { + const uint32_t cur_m_rows = MIN(src1_nrows - m_start, m_chunk); + mmctx->cur_m_start = m_start; + mmctx->cur_m_rows = cur_m_rows; + + if (need_quant) { + const uint32_t quant_tasks = MIN(cur_m_rows, octx->n_threads); + mmctx->n_quant_rows_per_thread = (cur_m_rows + quant_tasks - 1) / quant_tasks; + mmctx->n_quant_tasks = quant_tasks; + atomic_store(&mmctx->quant_barrier, quant_tasks); + mmctx->quant_task_func = quant_task_func; + } else { + mmctx->quant_task_func = NULL; + mmctx->n_quant_tasks = 0; + } + + worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, octx->n_threads); + } } else { - mmctx->quant_task_func = NULL; - mmctx->n_quant_tasks = 0; - } + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = src1_nrows; + + if (need_quant) { + mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->quant_task_func = quant_task_func; + mmctx->n_quant_tasks = n_quant_tasks; + atomic_init(&mmctx->quant_barrier, n_quant_tasks); + } else { + mmctx->quant_task_func = NULL; + mmctx->n_quant_tasks = 0; + } - htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, octx->n_threads); + worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, octx->n_threads); + } return HTP_STATUS_OK; } @@ -1567,7 +1829,6 @@ static void hvx_mm_nx_2d(unsigned int nth, unsigned int ith, void * data) { uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; uint8_t * restrict src1_data = mmctx->vtcm_src1; - dma_queue * dma_queue = octx->ctx->dma[ith]; const uint32_t n_prefetch = kparams->n_prefetch; assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); const uint32_t prefetch_mask = n_prefetch - 1; @@ -1580,16 +1841,25 @@ static void hvx_mm_nx_2d(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict src_w = octx->src[widx]; const struct htp_tensor * restrict dst = octx->dsts[widx]; if (!src_w || !dst) continue; + dma_queue * dma_q = octx->ctx->dma[ith]; const uint32_t ne00 = src_w->ne[0]; const uint32_t ne01 = src_w->ne[1]; - const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3]; + uint32_t start_row = 0; + uint32_t end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + start_row = range.start; + end_row = range.start + range.count; + } - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); + const uint32_t nrows = end_row - start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); src0_nrows_per_thread += (src0_nrows_per_thread & 1); - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); if (src0_start_row >= src0_end_row) continue; @@ -1597,17 +1867,17 @@ static void hvx_mm_nx_2d(unsigned int nth, unsigned int ith, void * data) { const size_t src0_row_size = src_w->nb[1]; const size_t src0_stride = hex_round_up(src0_row_size, 128); - const uint8_t * restrict src0_row = (const uint8_t *) src_w->data; + const dma_addr_t src0_row = src_w->data; for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { const int is0 = (ir0 - src0_start_row); if (is0 >= (int)n_prefetch) break; - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), src0_stride, src0_row_size, src0_row_size, 2); } for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); uint32_t ir1 = 0; for (; ir1 + 1 < src1_nrows; ir1 += 2) { @@ -1627,7 +1897,7 @@ static void hvx_mm_nx_2d(unsigned int nth, unsigned int ith, void * data) { const int pr0 = (ir0 + n_prefetch); const int is0 = (pr0 - src0_start_row) & prefetch_mask; if (pr0 < src0_end_row_x2) { - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), src0_stride, src0_row_size, src0_row_size, 2); } } @@ -1635,9 +1905,9 @@ static void hvx_mm_nx_2d(unsigned int nth, unsigned int ith, void * data) { if (src0_end_row != src0_end_row_x2) { uint32_t ir0 = src0_end_row_x2; const int is0 = (ir0 - src0_start_row) & prefetch_mask; - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), src0_stride, src0_row_size, src0_row_size, 1); - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) { const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); @@ -1667,6 +1937,7 @@ DEQUANTIZE_WORKER_LOOP_IMPL(q4_1) DEQUANTIZE_WORKER_LOOP_IMPL(iq4_nl) DEQUANTIZE_WORKER_LOOP_IMPL(mxfp4) DEQUANTIZE_WORKER_LOOP_IMPL(q8_0) +DEQUANTIZE_WORKER_LOOP_IMPL(q6_k) static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data) { tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; @@ -1773,7 +2044,7 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( // Push step 0 if (n_steps > 0 && n_rows > 0) { uint32_t nrows_to_fetch = hex_smin(n_rows, R); - dma_queue_push(dma_q, dma_make_ptr(thread_f32_act, src + c_first), + dma_queue_push(dma_q, dma_make_data(thread_f32_act, src + c_first), c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); } // Push step 1 @@ -1783,7 +2054,7 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); const float *next_src = src + next_r * k_stride + c_first; float *next_buf = thread_f32_act + 1 * R * c_len; - dma_queue_push(dma_q, dma_make_ptr(next_buf, next_src), + dma_queue_push(dma_q, dma_make_data(next_buf, next_src), c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); } } @@ -1814,7 +2085,7 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( if (next_r < n_rows) { uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); const float *next_src = src + next_r * k_stride + c_first; - dma_queue_push(dma_q, dma_make_ptr(curr_buf, next_src), + dma_queue_push(dma_q, dma_make_data(curr_buf, next_src), c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); } } @@ -1918,7 +2189,7 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( // Push step 0 if (n_steps > 0 && n_rows > 0) { uint32_t nrows_to_fetch = hex_smin(n_rows, R); - dma_queue_push(dma_q, dma_make_ptr(thread_f32_act, src), + dma_queue_push(dma_q, dma_make_data(thread_f32_act, src), k_block * sizeof(float), k_stride * sizeof(float), k_valid * sizeof(float), nrows_to_fetch); } // Push step 1 (if valid) @@ -1928,7 +2199,7 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); const float *next_src = src + next_r * k_stride; float *next_buf = thread_f32_act + 1 * R * k_block; - dma_queue_push(dma_q, dma_make_ptr(next_buf, next_src), + dma_queue_push(dma_q, dma_make_data(next_buf, next_src), k_block * sizeof(float), k_stride * sizeof(float), k_valid * sizeof(float), nrows_to_fetch); } } @@ -1957,7 +2228,7 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( if (next_r < n_rows) { uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); const float *next_src = src + next_r * k_stride; - dma_queue_push(dma_q, dma_make_ptr(curr_buf, next_src), + dma_queue_push(dma_q, dma_make_data(curr_buf, next_src), k_block * sizeof(float), k_stride * sizeof(float), k_valid * sizeof(float), nrows_to_fetch); } } @@ -2364,10 +2635,12 @@ static inline void hmx_matmul_job_init(hmx_matmul_job_t * job, } static int hmx_mm_2d_f32(struct htp_context *ctx, + dma_queue *weight_dma, float *restrict dst, - const float *restrict src2, + dma_addr_t src2_addr, + size_t src2_bytes, const float *activation, - const uint8_t *weight, + dma_addr_t weight, int m, int k, int n, int act_stride, int weight_stride, @@ -2401,9 +2674,11 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, switch (weight_type) { case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break; case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break; - case HTP_TYPE_Q4_1: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break; + case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break; case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break; case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break; + case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break; case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break; case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break; default: @@ -2428,7 +2703,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const size_t qweight_row_stride = is_quant ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, false, pipeline, act_threads, aligned_tile_size); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, false, pipeline, act_threads, aligned_tile_size, src2_bytes); vtcm_used = L.total_bytes; if (vtcm_used > vtcm_budget) { @@ -2453,6 +2728,13 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 + const bool has_src2 = (src2_bytes > 0 && src2_addr != 0); + float *vtcm_src2 = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_src2, has_src2); + if (has_src2) { + dma_queue_push(weight_dma, dma_make_data(vtcm_src2, src2_addr), hex_align_up(src2_bytes, 128), 0, src2_bytes, 1); + dma_queue_pop(weight_dma); + } + FARF(HIGH, "hmx-mm-2d: m %d k %d n %d wtype %d mc %zu nc %zu vtcm %zu/%zu", m, k, n, weight_type, m_chunk_n_rows, n_chunk_n_cols, vtcm_used, vtcm_budget); @@ -2489,13 +2771,13 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, // Prologue: push A0 and optionally A1 (if n_chunk_cnt > 1) const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); const uint32_t height_A0 = is_quant ? (n_cols_A0 / 32) * n_k_tiles : n_cols_A0; - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[0], weight), + dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0); if (1 < n_chunk_cnt) { const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); const uint32_t height_A1 = is_quant ? (n_cols_A1 / 32) * n_k_tiles : n_cols_A1; - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[1], weight + n_chunk_n_cols * weight_stride), + dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[1], weight + n_chunk_n_cols * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1); } @@ -2508,7 +2790,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const size_t n_cols_p2 = hex_smin(n - nc_p2, n_chunk_n_cols); // 1. pop A_i - void * curr_raw = dma_queue_pop(ctx->dma[0]).dst; + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; // 2. dequantize A_i dequantize_tiled_weight_chunk_to_fp16_tiles( @@ -2519,7 +2801,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, // 3. push A_{i+2} (if i+2 < n_chunk_cnt) if (i + 2 < n_chunk_cnt) { const uint32_t height_p2 = is_quant ? (n_cols_p2 / 32) * n_k_tiles : n_cols_p2; - dma_queue_push(ctx->dma[0], dma_make_ptr(curr_raw, weight + nc_p2 * weight_stride), + dma_queue_push(weight_dma, dma_make_data(curr_raw, weight + nc_p2 * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2); } @@ -2536,7 +2818,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const size_t nc_prev = (i - 1) * n_chunk_n_cols; const size_t n_cols_prev = hex_smin(n - nc_prev, n_chunk_n_cols); float *output_chunk = dst + (mr * dst_stride + nc_prev); - const float *src2_chunk = src2 ? (src2 + mr * src2_stride + nc_prev) : NULL; + const float *src2_chunk = has_src2 ? (vtcm_src2 + mr * src2_stride + nc_prev) : NULL; int chunk_dst_cols = dst_cols - (int)nc_prev; if (chunk_dst_cols > 0) { transfer_output_chunk_threaded(ctx, output_chunk, src2_chunk, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, src2_stride, chunk_dst_cols, n_threads); @@ -2549,7 +2831,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const size_t nc_last = (n_chunk_cnt - 1) * n_chunk_n_cols; const size_t n_cols_last = hex_smin(n - nc_last, n_chunk_n_cols); float *output_chunk = dst + (mr * dst_stride + nc_last); - const float *src2_chunk = src2 ? (src2 + mr * src2_stride + nc_last) : NULL; + const float *src2_chunk = has_src2 ? (vtcm_src2 + mr * src2_stride + nc_last) : NULL; int chunk_dst_cols = dst_cols - (int)nc_last; if (chunk_dst_cols > 0) { transfer_output_chunk_threaded(ctx, output_chunk, src2_chunk, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, src2_stride, chunk_dst_cols, n_threads); @@ -2581,7 +2863,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, if (n > 0) { const size_t n_cols = hex_smin(n, n_chunk_n_cols); const uint32_t height = is_quant ? (n_cols / 32) * n_k_tiles : n_cols; - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); + dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); } for (size_t nc = 0; nc < n; nc += n_chunk_n_cols) { @@ -2590,7 +2872,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); // A: Wait for weight DMA - void * curr_raw = dma_queue_pop(ctx->dma[0]).dst; + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; // B: Weight Dequantize (Threaded) dequantize_tiled_weight_chunk_to_fp16_tiles( @@ -2603,7 +2885,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, if (nc_next < n) { const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols); const uint32_t height_next = is_quant ? (n_cols_next / 32) * n_k_tiles : n_cols_next; - dma_queue_push(ctx->dma[0], dma_make_ptr(curr_raw, weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); + dma_queue_push(weight_dma, dma_make_data(curr_raw, weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); } // C: HMX Compute (Queue-based) @@ -2613,7 +2895,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, // D: Output Store float *output_chunk = dst + (mr * dst_stride + nc); - const float *src2_chunk = src2 ? (src2 + mr * src2_stride + nc) : NULL; + const float *src2_chunk = has_src2 ? (vtcm_src2 + mr * src2_stride + nc) : NULL; int chunk_dst_cols = dst_cols - (int)nc; if (chunk_dst_cols > 0) { transfer_output_chunk_threaded(ctx, output_chunk, src2_chunk, vtcm_output, n_rows, n_cols, dst_stride, src2_stride, chunk_dst_cols, n_threads); @@ -2638,10 +2920,6 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const struct htp_tensor * restrict src0 = octx->src[0]; const struct htp_tensor * restrict act = octx->src[n_weights]; - if (!src0 || !act) { - return HTP_STATUS_INVAL_PARAMS; - } - const int weight_type = (int) src0->type; const int k = (int) act->ne[0]; const int k_valid = (int) act->ne[0]; @@ -2661,9 +2939,11 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k switch (weight_type) { case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break; case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break; - case HTP_TYPE_Q4_1: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break; + case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break; case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break; case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break; + case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break; case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break; case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break; default: @@ -2690,7 +2970,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const uint32_t dma_width_bytes = is_quant ? tile_size : row_stride; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, false, pipeline, act_threads, aligned_tile_size); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, false, pipeline, act_threads, aligned_tile_size, 0); if (L.total_bytes > vtcm_budget) { FARF(ERROR, "hmx-mm-nx-2d: VTCM overflow: used %zu budget %zu, m %d k %d mc %d nc %d", @@ -2714,16 +2994,31 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 - FARF(HIGH, "hmx-mm-nx-2d: n_weights %u m %d k %d wtype %d mc %d nc %d vtcm %zu/%zu", - n_weights, m, k, weight_type, m_chunk_n_rows, n_chunk_n_cols, L.total_bytes, vtcm_budget); + int m_start = 0; + int m_rows = m; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(octx->dsts[0], sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_rows = (int) range.count; + } + + if (m_rows == 0) { + return HTP_STATUS_OK; + } + + FARF(HIGH, "hmx-mm-nx-2d: n_weights %u m %d (%d..%d) k %d wtype %d mc %d nc %d vtcm %zu/%zu", + n_weights, m, m_start, m_start + m_rows, k, weight_type, m_chunk_n_rows, n_chunk_n_cols, L.total_bytes, vtcm_budget); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + const size_t mr_end = (size_t)(m_start + m_rows); + if (pipeline) { hmx_matmul_job_t job_slots[2]; - for (size_t mr = 0; mr < (size_t) m; mr += m_chunk_n_rows) { - const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows); void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 }; void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; @@ -2749,7 +3044,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - const uint8_t * weight = (const uint8_t *) src_w->data; + const dma_addr_t weight = src_w->data; + dma_queue * weight_dma = octx->ctx->dma[0]; float * dst_ptr = (float *) dst->data; const size_t n = src_w->ne[1]; if (n == 0) continue; @@ -2762,13 +3058,13 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); const uint32_t height_A0 = is_quant ? (n_cols_A0 / 32) * n_k_tiles : n_cols_A0; - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[0], weight), + dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0); if (1 < n_chunk_cnt) { const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); const uint32_t height_A1 = is_quant ? (n_cols_A1 / 32) * n_k_tiles : n_cols_A1; - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[1], weight + n_chunk_n_cols * weight_stride), + dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[1], weight + n_chunk_n_cols * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1); } @@ -2779,7 +3075,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); const size_t n_cols_p2 = hex_smin(n - nc_p2, n_chunk_n_cols); - void * curr_raw = dma_queue_pop(ctx->dma[0]).dst; + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_weight_bufs[i % 2], curr_raw, @@ -2788,7 +3084,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k if (i + 2 < n_chunk_cnt) { const uint32_t height_p2 = is_quant ? (n_cols_p2 / 32) * n_k_tiles : n_cols_p2; - dma_queue_push(ctx->dma[0], dma_make_ptr(curr_raw, weight + nc_p2 * weight_stride), + dma_queue_push(weight_dma, dma_make_data(curr_raw, weight + nc_p2 * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2); } @@ -2822,8 +3118,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k } } else { hmx_matmul_job_t job; - for (size_t mr = 0; mr < (size_t) m; mr += m_chunk_n_rows) { - const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows); struct activation_transfer_params act_params = { .ctx = ctx, @@ -2846,7 +3142,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - const uint8_t * weight = (const uint8_t *) src_w->data; + const dma_addr_t weight = src_w->data; + dma_queue * weight_dma = octx->ctx->dma[0]; float * dst_ptr = (float *) dst->data; const size_t n = src_w->ne[1]; if (n == 0) continue; @@ -2859,7 +3156,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k if (n > 0) { const size_t n_cols = hex_smin(n, n_chunk_n_cols); const uint32_t height = is_quant ? (n_cols / 32) * n_k_tiles : n_cols; - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); + dma_queue_push(weight_dma, dma_make_data(vtcm_weight_raw[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); } for (size_t nc = 0; nc < n; nc += n_chunk_n_cols) { @@ -2867,7 +3164,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const size_t n_row_tiles = hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS); const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); - void * curr_raw = dma_queue_pop(ctx->dma[0]).dst; + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_scratch0, curr_raw, @@ -2878,7 +3175,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k if (nc_next < n) { const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols); const uint32_t height_next = is_quant ? (n_cols_next / 32) * n_k_tiles : n_cols_next; - dma_queue_push(ctx->dma[0], dma_make_ptr(curr_raw, weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); + dma_queue_push(weight_dma, dma_make_data(curr_raw, weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); } hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, vtcm_scratch0, vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); @@ -2898,13 +3195,11 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k return HTP_STATUS_OK; } -static inline const __fp16 *hmx_mm_weight_batch_ptr(const hmx_mm_f16_f32_batched_params_t *params, - int dst_b2, int dst_b3) { +static inline dma_addr_t hmx_mm_weight_batch_data(const hmx_mm_f16_f32_batched_params_t *params, + int dst_b2, int dst_b3) { const size_t b2_idx = (params->r2 <= 1) ? (size_t) dst_b2 : (size_t) fastdiv((uint32_t) dst_b2, ¶ms->div_r2); const size_t b3_idx = (params->r3 <= 1) ? (size_t) dst_b3 : (size_t) fastdiv((uint32_t) dst_b3, ¶ms->div_r3); - return (const __fp16 *) ((const uint8_t *) params->weight + - b2_idx * params->src0_nb2 + - b3_idx * params->src0_nb3); + return params->weight + b2_idx * params->src0_nb2 + b3_idx * params->src0_nb3; } static inline const float *hmx_mm_activation_batch_ptr(const hmx_mm_f16_f32_batched_params_t *params, @@ -2921,13 +3216,6 @@ static inline float *hmx_mm_dst_batch_ptr(const hmx_mm_f16_f32_batched_params_t (size_t) dst_b3 * params->dst_nb3); } -static inline const float *hmx_mm_src2_batch_ptr(const hmx_mm_f16_f32_batched_params_t *params, - int src2_b2, int src2_b3) { - return params->src2 ? (const float *) ((const uint8_t *) params->src2 + - (size_t) src2_b2 * params->src2_nb2 + - (size_t) src2_b3 * params->src2_nb3) : NULL; -} - static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx, const hmx_mm_f16_f32_batched_params_t *params, int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size, @@ -2935,15 +3223,18 @@ static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx, int ret = 0; for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) { for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) { - ret = hmx_mm_2d_f32(ctx, hmx_mm_dst_batch_ptr(params, b2, b3), - hmx_mm_src2_batch_ptr(params, b2, b3), - hmx_mm_activation_batch_ptr(params, b2, b3), - (const uint8_t *)hmx_mm_weight_batch_ptr(params, b2, b3), - params->m, params->k, params->n, - params->act_stride, params->weight_stride * (int)sizeof(__fp16), - HTP_TYPE_F16, params->k, params->dst_stride, params->src2_stride, params->n, - m_chunk, n_chunk, pipeline, n_threads, act_threads, - act_threads_div, k_div, 0, 0, vtcm_size); + dma_addr_t cur_src2_addr = params->src2_addr ? (params->src2_addr + + (dma_addr_t) b2 * params->src2_nb2 + + (dma_addr_t) b3 * params->src2_nb3) : 0; + ret = hmx_mm_2d_f32(ctx, params->weight_dma, hmx_mm_dst_batch_ptr(params, b2, b3), + cur_src2_addr, params->src2_bytes, + hmx_mm_activation_batch_ptr(params, b2, b3), + hmx_mm_weight_batch_data(params, b2, b3), + params->m, params->k, params->n, + params->act_stride, params->weight_stride * (int)sizeof(__fp16), + HTP_TYPE_F16, params->k, params->dst_stride, params->src2_stride, params->n, + m_chunk, n_chunk, pipeline, n_threads, act_threads, + act_threads_div, k_div, 0, 0, vtcm_size); } } return ret; @@ -2985,7 +3276,7 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ size_t vtcm_used = vtcm_size; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_F16_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, use_dma_activation, false, act_threads, 0); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_F16_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, use_dma_activation, false, act_threads, 0, params->src2_bytes); if (L.total_bytes > vtcm_budget) { FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to simple batched loop", __func__); @@ -3002,6 +3293,13 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ __fp16 *vtcm_scales = VTCM_LAYOUT_PTR(__fp16, base, L.off_scales); float *vtcm_f32_act = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_act_f32, use_dma_activation); + const bool has_src2 = (params->src2_bytes > 0 && params->src2_addr != 0); + float *vtcm_src2 = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_src2, has_src2); + if (has_src2) { + dma_queue_push(params->weight_dma, dma_make_data(vtcm_src2, params->src2_addr), hex_align_up(params->src2_bytes, 128), 0, params->src2_bytes, 1); + dma_queue_pop(params->weight_dma); + } + hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 FARF(HIGH, "%s: grouped path m=%d k=%d n=%d group=%d streams=%d mc=%zu nc=%zu vtcm=%zu/%zu", @@ -3018,7 +3316,8 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ for (int b3 = 0; b3 < params->ne13; ++b3) { for (int b2_base = 0; b2_base < params->ne12; b2_base += group_size) { - const __fp16 *weight_group = hmx_mm_weight_batch_ptr(params, b2_base, b3); + const dma_addr_t weight_group = hmx_mm_weight_batch_data(params, b2_base, b3); + dma_queue * weight_dma = params->weight_dma; for (size_t mr = 0; mr < (size_t) params->m; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin((size_t) params->m - mr, m_chunk_n_rows); @@ -3052,12 +3351,12 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ // Prologue: Push A0 and A1 (if exists) { const size_t n_cols_first = hex_smin((size_t) params->n, n_chunk_n_cols); - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_scratch0, weight_group), + dma_queue_push(weight_dma, dma_make_data(vtcm_scratch0, weight_group), fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_first); } if (n_chunk_n_cols < (size_t) params->n) { const size_t n_cols_second = hex_smin((size_t) params->n - n_chunk_n_cols, n_chunk_n_cols); - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_scratch1, weight_group + params->weight_stride), + dma_queue_push(weight_dma, dma_make_data(vtcm_scratch1, weight_group + params->weight_stride * sizeof(__fp16)), fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_second); } @@ -3066,16 +3365,16 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ const size_t n_col_tiles = hmx_ceil_div((int) n_cols, HTP_MM_HMX_TILE_N_COLS); { - void * curr_raw = dma_queue_pop(ctx->dma[0]).dst; + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; hmx_interleave_rows_to_tiles(vtcm_weight, (const __fp16 *) curr_raw, n_cols, params->k, params->k, 0, n_cols); const size_t nc_next = nc + n_chunk_n_cols * 2; if (nc_next < (size_t) params->n) { const size_t n_cols_next = hex_smin((size_t) params->n - nc_next, n_chunk_n_cols); - const __fp16 *next_weight_chunk = weight_group + nc_next * params->weight_stride; + const dma_addr_t next_weight_chunk = weight_group + nc_next * params->weight_stride * sizeof(__fp16); - dma_queue_push(ctx->dma[0], dma_make_ptr(curr_raw, next_weight_chunk), + dma_queue_push(weight_dma, dma_make_data(curr_raw, next_weight_chunk), fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_next); } } @@ -3091,11 +3390,11 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ { float *output = hmx_mm_dst_batch_ptr(params, b2_base + g, b3) + mr * params->dst_stride + nc; - const float *src2_chunk = params->src2 ? (hmx_mm_src2_batch_ptr(params, b2_base + g, b3) + mr * params->src2_stride + nc) : NULL; + const float *src2_chunk = has_src2 ? (vtcm_src2 + mr * params->src2_stride + nc) : NULL; int chunk_dst_cols = params->n - (int)nc; if (chunk_dst_cols > 0) { transfer_output_chunk_threaded(ctx, output, src2_chunk, vtcm_output, (int) n_rows, (int) n_cols, - params->dst_stride, params->src2_stride, chunk_dst_cols, ctx->n_threads); + params->dst_stride, params->src2_stride, chunk_dst_cols, n_threads); } } } @@ -3204,9 +3503,10 @@ static void transfer_output_chunk_scattered_threaded( } static int hmx_mm_id_2d_f32(struct htp_context *ctx, + dma_queue *weight_dma, float *restrict dst, const float *activation, - const uint8_t *weight, + dma_addr_t weight, int m, int k, int n, int k_valid, int ne11, @@ -3216,7 +3516,10 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, int weight_type, const struct mmid_row_mapping *matrix_rows, int cur_a, - int mapping_stride) { + int mapping_stride, + int m_start, + int m_end, + int n_threads) { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); @@ -3235,9 +3538,11 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, switch (weight_type) { case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break; case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break; - case HTP_TYPE_Q4_1: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break; + case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break; case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break; case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break; + case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break; case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break; case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break; default: @@ -3247,7 +3552,6 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); - const int n_threads = ctx->n_threads; const bool is_quant = (weight_type != HTP_TYPE_F16 && weight_type != HTP_TYPE_F32); const size_t vec_dot_size = k * sizeof(__fp16); @@ -3303,8 +3607,8 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, hmx_matmul_job_t job; - for (size_t mr = 0; mr < (size_t) m_padded; mr += m_chunk_n_rows) { - const size_t n_rows = hex_smin(m_padded - mr, m_chunk_n_rows); + for (size_t mr = (size_t) m_start; mr < (size_t) m_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin((size_t) m_end - mr, m_chunk_n_rows); const size_t n_row_tiles = hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS); transfer_activation_chunk_gathered_threaded( @@ -3315,7 +3619,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, if (n > 0) { const size_t n_cols = hex_smin((size_t) n, n_chunk_n_cols); const uint32_t height = is_quant ? (n_cols / 32) * n_k_tiles : n_cols; - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight, weight), + dma_queue_push(weight_dma, dma_make_data(vtcm_weight, weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); } @@ -3324,7 +3628,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); // A: Wait for weight DMA - void * curr_raw = dma_queue_pop(ctx->dma[0]).dst; + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; // B: Weight Dequantize (Threaded) dequantize_tiled_weight_chunk_to_fp16_tiles( @@ -3338,7 +3642,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, if (nc_next < (size_t) n) { const size_t n_cols_next = hex_smin((size_t) n - nc_next, n_chunk_n_cols); const uint32_t height_next = is_quant ? (n_cols_next / 32) * n_k_tiles : n_cols_next; - dma_queue_push(ctx->dma[0], dma_make_ptr(curr_raw, weight + nc_next * weight_stride), + dma_queue_push(weight_dma, dma_make_data(curr_raw, weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); } @@ -3368,31 +3672,52 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k const int act_stride = (int)(src1->nb[1] / sizeof(float)); const int wgt_stride = (int)(src0->nb[1] / sizeof(__fp16)); - const float * src2_ptr = NULL; + int m_start = 0; + int m_rows = m_total; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_total, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_rows = (int) range.count; + } + + if (m_rows == 0) { + return HTP_STATUS_OK; + } + + dma_addr_t src2_addr = 0; + size_t src2_bytes = 0; uint32_t src2_stride = 0; size_t src2_nb2 = 0; size_t src2_nb3 = 0; if (src2) { - src2_ptr = (const float *) src2->data; src2_stride = (src2->ne[1] == 1) ? 0 : (uint32_t) (src2->nb[1] / sizeof(float)); + src2_addr = src2->data + (dma_addr_t) m_start * src2_stride * sizeof(float); + src2_bytes = (size_t) kparams->vtcm_src2_size; src2_nb2 = (src2->ne[2] == 1) ? 0 : src2->nb[2]; src2_nb3 = (src2->ne[3] == 1) ? 0 : src2->nb[3]; } + const int dst_stride = (int)(dst->nb[1] / sizeof(float)); + float * dst_ptr = (float *) dst->data + m_start * dst_stride; + const float * act_ptr = (const float *) src1->data + m_start * act_stride; + int ret = -1; - const int n_threads = MIN(kparams->n_threads, (int) octx->n_threads); + const int n_threads = kparams->n_threads; if (kparams->kernel_type == HTP_MM_KERNEL_HMX_F16_BATCHED) { hmx_mm_f16_f32_batched_params_t batch_params = { - .dst = (float *) dst->data, - .src2 = src2_ptr, - .activation = (float *) src1->data, - .weight = (const __fp16 *) src0->data, - .m = m_total, + .dst = dst_ptr, + .src2_addr = src2_addr, + .src2_bytes = src2_bytes, + .activation = act_ptr, + .weight = src0->data, + .weight_dma = octx->ctx->dma[0], + .m = m_rows, .k = k, .n = n, .act_stride = act_stride, .weight_stride = wgt_stride, - .dst_stride = (int) (dst->nb[1] / sizeof(float)), + .dst_stride = dst_stride, .src2_stride = src2_stride, .ne02 = ne02, .ne03 = ne03, @@ -3420,9 +3745,9 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k kparams->vtcm_size); } else { ret = hmx_mm_2d_f32( - octx->ctx, (float*) dst->data, src2_ptr, (float*) src1->data, (const uint8_t *) src0->data, - m_total, k, n, act_stride, (int) src0->nb[1], (int) src0->type, (int) src1->ne[0], - (int)(dst->nb[1] / sizeof(float)), src2_stride, (int)dst->ne[0], + octx->ctx, octx->ctx->dma[0], dst_ptr, src2_addr, src2_bytes, act_ptr, src0->data, + m_rows, k, n, act_stride, (int) src0->nb[1], (int) src0->type, (int) src1->ne[0], + dst_stride, src2_stride, (int)dst->ne[0], kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, kparams->n_act_threads, &kparams->div_n_act_threads, @@ -3441,6 +3766,11 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k int op_matmul(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + if (kparams->n_hmx) { return hmx_mm_op_matmul(octx, kparams); } @@ -3463,15 +3793,26 @@ static int hmx_mm_op_matmul_id( const int32_t cne1 = matrix_row_counts[cur_a]; if (cne1 == 0) continue; - int ret = hmx_mm_id_2d_f32(octx->ctx, (float*) dst->data, (float*) src1->data, - (const uint8_t *) src0->data + cur_a * nb02, + const int m_padded = hex_align_up(cne1, 32); + int m_start = 0, m_end = m_padded; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && (uint32_t) cne1 >= octx->ctx->mdev.count; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_end = (int) (range.start + range.count); + } + if (m_start >= m_end) continue; + + int ret = hmx_mm_id_2d_f32(octx->ctx, octx->ctx->dma[0], (float*) dst->data, (float*) src1->data, + src0->data + cur_a * nb02, cne1, ne00, ne01, ne10, ne11, nb11, nb12, nb1, nb2, (int) src0->nb[1], (int) src0->type, - matrix_rows, cur_a, mmctx->mapping_stride); + matrix_rows, cur_a, mmctx->mapping_stride, + m_start, m_end, (int) octx->n_threads); if (ret != 0) { FARF(ERROR, "HMX matmul failed for expert %u, error %d\n", cur_a, ret); return HTP_STATUS_NO_SUPPORT; @@ -3505,7 +3846,7 @@ static int hvx_mm_matmul_id( uint32_t n_quant_tasks = 1; if (src1_nrows < octx->n_threads) { n_quant_tasks = MIN(total_nb, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; + quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; @@ -3516,15 +3857,15 @@ static int hvx_mm_matmul_id( } } else { n_quant_tasks = MIN(src1_nrows, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; + quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; } - size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads, 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; FARF(HIGH, "matmul-id-%s : src0-spad-size %zu src1-spad-size %zu src2-spad-size 0 dst-spad-size %zu (%zu)\n", mmctx->type, L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size); @@ -3554,10 +3895,13 @@ static int hvx_mm_matmul_id( mmctx->vtcm_src0_stride = src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); mmctx->vtcm_src1_size_per_thread = L.src1_bytes; mmctx->vtcm_src2_size_per_thread = 0; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = src1_nrows; mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; @@ -3587,20 +3931,35 @@ static int hmx_mm_op_matmul_id_nx( const int32_t cne1 = matrix_row_counts[cur_a]; if (cne1 == 0) continue; + const int m_padded = hex_align_up(cne1, 32); + int m_start = 0, m_end = m_padded; + if (octx->ctx->mdev.count > 1) { + bool can_split = (uint32_t) cne1 >= octx->ctx->mdev.count; + for (uint32_t p = 0; p < n_weights && can_split; ++p) { + const struct htp_tensor * restrict dst = octx->dsts[p]; + can_split = !dst || htp_tensor_mdev_data_aligned(dst); + } + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_end = (int) (range.start + range.count); + } + if (m_start >= m_end) continue; + for (uint32_t p = 0; p < n_weights; ++p) { const struct htp_tensor * restrict src_w = octx->src[p]; const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - int ret = hmx_mm_id_2d_f32(octx->ctx, (float*) dst->data, (float*) act->data, - (const uint8_t *) src_w->data + cur_a * src_w->nb[2], + int ret = hmx_mm_id_2d_f32(octx->ctx, octx->ctx->dma[0], (float*) dst->data, (float*) act->data, + src_w->data + cur_a * src_w->nb[2], cne1, src_w->ne[0], src_w->ne[1], act->ne[0], act->ne[1], act->nb[1], act->nb[2], dst->nb[1], dst->nb[2], (int) src_w->nb[1], (int) src_w->type, - matrix_rows, cur_a, mmctx->mapping_stride); + matrix_rows, cur_a, mmctx->mapping_stride, + m_start, m_end, (int) octx->n_threads); if (ret != 0) { FARF(ERROR, "HMX matmul ID NX failed for expert %u weight %u, error %d\n", cur_a, p, ret); return HTP_STATUS_NO_SUPPORT; @@ -3637,7 +3996,7 @@ static int hvx_mm_matmul_id_nx( uint32_t n_quant_tasks = 1; if (src1_nrows < octx->n_threads) { n_quant_tasks = MIN(total_nb, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; + quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; @@ -3648,15 +4007,15 @@ static int hvx_mm_matmul_id_nx( } } else { n_quant_tasks = MIN(src1_nrows, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; + quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; } - size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]); + size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]); struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads, 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; if (octx->ctx->vtcm_size < vtcm_size) { FARF(ERROR, "matmul-id-nx: current VTCM reservation %zu is too small, needed %zu\n", @@ -3678,9 +4037,12 @@ static int hvx_mm_matmul_id_nx( mmctx->vtcm_src0_stride = 0; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); mmctx->vtcm_src1_size_per_thread = L.src1_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = src1_nrows; mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; @@ -3769,17 +4131,25 @@ static inline void scan_expert_ids( int op_matmul_id(struct htp_ops_context * octx) { htp_matmul_tensors_preamble; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + struct htp_mm_context mmctx_struct = {0}; + struct htp_mm_context * mmctx = &mmctx_struct; + + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + struct htp_thread_trace * tr = &octx->ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - struct htp_mm_context mmctx_struct = {0}; - struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; mmctx->act = src1; - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - const struct htp_tensor * restrict ids = octx->src[2]; + if (htp_tensor_is_extended(ids) || htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } const size_t src0_row_size = nb01; const size_t dst_row_size = nb1; @@ -3789,9 +4159,6 @@ int op_matmul_id(struct htp_ops_context * octx) { const uint32_t src0_nrows = ne01; // per expert const uint32_t src1_nrows = ne11 * ne12 * ne13; - mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div); - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); - // row groups const int n_ids = ids->ne[0]; // n_expert_used const int n_as = ne02; // n_expert @@ -3833,9 +4200,11 @@ int op_matmul_id(struct htp_ops_context * octx) { mmctx->matrix_row_counts = matrix_row_counts; mmctx->matrix_rows = matrix_rows; mmctx->mapping_stride = mapping_stride; - mmctx->mm_div_ne11 = kparams->div_ne11; + mmctx->mm_div_ne11 = kparams->div_ne1; mmctx->src0_row_size_padded = src0_row_size_padded; mmctx->src1_nrows = src1_nrows; + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = src1_nrows; htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); @@ -3843,6 +4212,29 @@ int op_matmul_id(struct htp_ops_context * octx) { if (kparams->n_hmx) { s = hmx_mm_op_matmul_id(octx, mmctx); } else { + uint32_t src0_row_start = 0; + uint32_t src0_row_end = src0_nrows; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + src0_row_start = range.start; + src0_row_end = range.start + range.count; + } + + if (src0_row_start >= src0_row_end) { + if (mapping_buf != octx->ctx->ddr_spad_base) { + free(mapping_buf); + } + return HTP_STATUS_OK; + } + + const uint32_t nrows = src0_row_end - src0_row_start; + mmctx->src0_row_start = src0_row_start; + mmctx->src0_row_end = src0_row_end; + + mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); + mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); + if (hvx_mm_init_vec_dot(mmctx, src0->type) == 0) { s = hvx_mm_matmul_id(octx, mmctx, src1_nrows > 1 ? hvx_mm_id : hvx_mv_id); } else { @@ -3858,29 +4250,39 @@ int op_matmul_id(struct htp_ops_context * octx) { } int op_matmul_id_nx(struct htp_ops_context * octx) { + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + struct htp_mm_context mmctx_struct = {0}; + struct htp_mm_context * mmctx = &mmctx_struct; + + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + struct htp_thread_trace * tr = &octx->ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + mmctx->octx = octx; const uint32_t n_weights = kparams->n_weights; const struct htp_tensor * restrict src0 = octx->src[0]; const struct htp_tensor * restrict act = octx->src[n_weights]; const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; + if (htp_tensor_is_extended(ids) || htp_tensor_is_extended(act)) { + return HTP_STATUS_NO_SUPPORT; + } + for (uint32_t p = 0; p < n_weights; p++) { + if (octx->dsts[p] && htp_tensor_is_extended(octx->dsts[p])) { + return HTP_STATUS_NO_SUPPORT; + } + } - struct htp_mm_context mmctx_struct = {0}; - struct htp_mm_context * mmctx = &mmctx_struct; - mmctx->octx = octx; mmctx->act = act; const size_t src0_row_size = src0->nb[1]; const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - const uint32_t src0_nrows = src0->ne[1]; const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3]; - mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div); - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); - const int n_ids = ids->ne[0]; const int n_as = src0->ne[2]; @@ -3921,9 +4323,11 @@ int op_matmul_id_nx(struct htp_ops_context * octx) { mmctx->matrix_row_counts = matrix_row_counts; mmctx->matrix_rows = matrix_rows; mmctx->mapping_stride = mapping_stride; - mmctx->mm_div_ne11 = kparams->div_ne11; + mmctx->mm_div_ne11 = kparams->div_ne1; mmctx->src0_row_size_padded = src0_row_size_padded; mmctx->src1_nrows = src1_nrows; + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = src1_nrows; htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); @@ -3946,6 +4350,12 @@ int op_matmul_id_nx(struct htp_ops_context * octx) { } int op_matmul_nx(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + if (kparams->n_hmx) { return hmx_mm_nx_2d_f32(octx, kparams); } @@ -3960,7 +4370,7 @@ int op_matmul_nx(struct htp_ops_context * octx) { bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL || - src0->type == HTP_TYPE_MXFP4); + src0->type == HTP_TYPE_MXFP4 || src0->type == HTP_TYPE_Q4_K); struct htp_mm_context mmctx_struct = {0}; struct htp_mm_context * mmctx = &mmctx_struct; @@ -3968,6 +4378,9 @@ int op_matmul_nx(struct htp_ops_context * octx) { mmctx->act = act; const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3]; + mmctx->src1_nrows = src1_nrows; + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = src1_nrows; const size_t src0_row_size = src0->nb[1]; const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); @@ -3982,12 +4395,9 @@ int op_matmul_nx(struct htp_ops_context * octx) { worker_callback_t quant_task_func; uint32_t n_quant_tasks = 1; - if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { - n_quant_tasks = MIN(src1_nrows, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat; - } else if (src1_nrows < octx->n_threads) { + if (src1_nrows < octx->n_threads) { n_quant_tasks = MIN(total_nb, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; + quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; @@ -3998,21 +4408,18 @@ int op_matmul_nx(struct htp_ops_context * octx) { } } else { n_quant_tasks = MIN(src1_nrows, octx->n_threads); - quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; + quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; } - size_t src1_row_size; - if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { - src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(act->ne[0]) : htp_mm_q8_0_flat_row_size(act->ne[0]); - } else { - src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]); - } + const size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) + ? htp_mm_q8_1_tiled_row_size(act->ne[0]) + : htp_mm_q8_0_tiled_row_size(act->ne[0]); struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads, 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, false, true); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; if (octx->ctx->vtcm_size < vtcm_size) { FARF(ERROR, "matmul-nx: current VTCM reservation %zu is too small, needed %zu\n", @@ -4034,9 +4441,9 @@ int op_matmul_nx(struct htp_ops_context * octx) { mmctx->vtcm_src0_stride = is_repacked ? 0 : src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); mmctx->vtcm_src1_size_per_thread = L.src1_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; @@ -4047,24 +4454,14 @@ int op_matmul_nx(struct htp_ops_context * octx) { const uint32_t n_matmul_jobs = octx->n_threads; worker_callback_t matmul_job_func; if (is_repacked) { - if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { - switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0_flat; break; - case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1_flat; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0_flat; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl_flat; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4_flat; break; - default: return HTP_STATUS_NO_SUPPORT; - } - } else { - switch (src0->type) { - case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0; break; - case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1; break; - case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0; break; - case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl; break; - case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4; break; - default: return HTP_STATUS_NO_SUPPORT; - } + switch (src0->type) { + case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0; break; + case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1; break; + case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0; break; + case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl; break; + case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4; break; + default: return HTP_STATUS_NO_SUPPORT; } } else { matmul_job_func = hvx_mm_nx_2d; diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h index 2dbcb0c2e51e..fe9dbb61cd98 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.h +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -25,6 +25,11 @@ extern "C" { #define HTP_MM_WEIGHT_TILE_SIZE_Q8_0 1088 #define HTP_MM_WEIGHT_TILE_SIZE_IQ4_NL 576 #define HTP_MM_WEIGHT_TILE_SIZE_MXFP4 544 +// Q6_K native 6-bit tile (32 rows x 32 k), vrmpy-ready: byte 4*row+b of a vector holds k = 4*group+b +// vectors 0..3: low nibbles, vector i holds group 2i (low nibble) and group 2i+1 (high nibble) +// vectors 4..5: high 2 bits, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6 +// vector 6: fp16 scales per row, d * scales[]: k 0..15 in lanes 0..31, k 16..31 in lanes 32..63 +#define HTP_MM_WEIGHT_TILE_SIZE_Q6_K 896 // --- Weight Repacked Aligned Tile Sizes --- #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0 640 @@ -32,6 +37,7 @@ extern "C" { #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q8_0 1152 #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_IQ4_NL 640 #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_MXFP4 640 +#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K 896 // --- Activation Tiled Block Sizes (including padding) --- #define HTP_MM_ACT_TILE_SIZE_Q8_0 1152 @@ -56,17 +62,11 @@ enum htp_mm_kernel_type { // HVX floating-point paths HTP_MM_KERNEL_HVX_F16_F16_VTCM, - HTP_MM_KERNEL_HVX_F16_F16_DDR, - HTP_MM_KERNEL_HVX_F16_F32_DDR, - HTP_MM_KERNEL_HVX_F32_F32_VTCM, - HTP_MM_KERNEL_HVX_F32_F32_DDR, - HTP_MM_KERNEL_HVX_F32_F16_DDR, // HVX quantized paths HTP_MM_KERNEL_HVX_QUANT_ROW, // standard row-wise parallel quantization HTP_MM_KERNEL_HVX_QUANT_BLOCK, // parallel block-wise quantization - HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT, // row-wise fallback flat quantization }; // Op-specific struct for precomputed matmul params @@ -95,7 +95,7 @@ struct htp_mm_kernel_params { struct fastdiv_values div_ne1; struct fastdiv_values div_r2; struct fastdiv_values div_r3; - struct fastdiv_values div_ne11; + struct fastdiv_values div_ne12; struct fastdiv_values div_n_act_threads; struct fastdiv_values div_ne00_padded; }; @@ -195,9 +195,12 @@ static inline uint32_t htp_mm_get_weight_tile_size(int weight_type) { case HTP_TYPE_IQ4_NL: return HTP_MM_WEIGHT_TILE_SIZE_Q4_0; case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: return HTP_MM_WEIGHT_TILE_SIZE_Q4_1; case HTP_TYPE_Q8_0: return HTP_MM_WEIGHT_TILE_SIZE_Q8_0; + case HTP_TYPE_Q6_K: + return HTP_MM_WEIGHT_TILE_SIZE_Q6_K; case HTP_TYPE_MXFP4: return HTP_MM_WEIGHT_TILE_SIZE_MXFP4; default: @@ -211,9 +214,12 @@ static inline uint32_t htp_mm_get_weight_aligned_tile_size(int weight_type) { case HTP_TYPE_IQ4_NL: return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0; case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_1; case HTP_TYPE_Q8_0: return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q8_0; + case HTP_TYPE_Q6_K: + return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K; case HTP_TYPE_MXFP4: return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_MXFP4; default: @@ -234,27 +240,15 @@ static inline size_t htp_mm_q8_1_tiled_row_size(uint32_t ne) { return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_1; } -static inline size_t htp_mm_q8_0_flat_row_size(uint32_t ne) { - const uint32_t quants_size = hex_align_up(ne, 128); - const uint32_t num_scales = (ne + 31) / 32; - const uint32_t scales_size = hex_align_up(num_scales * 2, 128); - return quants_size + scales_size; -} - -static inline size_t htp_mm_q8_1_flat_row_size(uint32_t ne) { - const uint32_t quants_size = hex_align_up(ne, 128); - const uint32_t num_scales = (ne + 31) / 32; - const uint32_t scales_size = hex_align_up(num_scales * 4, 128); - return quants_size + scales_size; -} - static inline size_t htp_mm_get_tiled_row_stride(int weight_type, uint32_t k) { uint32_t nb = (k + QK_Q4_0_TILED - 1) / QK_Q4_0_TILED; switch (weight_type) { case HTP_TYPE_Q4_0: case HTP_TYPE_IQ4_NL: case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: case HTP_TYPE_Q8_0: + case HTP_TYPE_Q6_K: case HTP_TYPE_MXFP4: return (size_t) nb * htp_mm_get_weight_tile_size(weight_type); case HTP_TYPE_F16: @@ -317,6 +311,7 @@ struct htp_mm_hmx_vtcm_layout { size_t off_dst[2]; // [1] is only used when pipelined size_t off_scratch[2]; // dequantization scratch pads size_t off_scales; // HMX scales (256 bytes) + size_t off_src2; // src2 bias in VTCM // Cached sizes of regions for HMX kernel use size_t weight_area_bytes; @@ -325,6 +320,7 @@ struct htp_mm_hmx_vtcm_layout { size_t output_area_bytes; size_t scratch_bytes[2]; size_t act_head_stride; + size_t src2_bytes; size_t total_bytes; }; @@ -358,7 +354,8 @@ static inline void htp_mm_hmx_vtcm_layout_build( bool use_dma_activation, bool pipeline, uint32_t act_threads, - uint32_t aligned_tile_size + uint32_t aligned_tile_size, + size_t src2_size ) { size_t off = 0; @@ -376,6 +373,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( size_t off_group_a = 0; VTCM_LAYOUT_ALLOC(off_group_a, off_act, activation_area_size); VTCM_LAYOUT_ALLOC(off_group_a, off_scales, HTP_MM_HMX_TILE_SIZE); // Padded to 2K for alignment and future persistent data + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_src2, hex_align_up(src2_size, HTP_MM_HMX_TILE_SIZE), src2_size > 0); // Group B: Compute-only buffers (starts at off_group_a) size_t off_group_b = off_group_a; @@ -404,6 +402,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( L->scratch_bytes[0] = scratch_area_size; L->scratch_bytes[1] = scratch_area_size; L->act_head_stride = act_head_stride; + L->src2_bytes = src2_size; off = off_group_a + hex_smax(group_b_size, group_c_size); } else { @@ -427,6 +426,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( size_t off_group_a = 0; VTCM_LAYOUT_ALLOC(off_group_a, off_scales, HTP_MM_HMX_TILE_SIZE); // Padded to 2K for alignment and future persistent data VTCM_LAYOUT_ALLOC(off_group_a, off_act, act_area_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_src2, hex_align_up(src2_size, HTP_MM_HMX_TILE_SIZE), src2_size > 0); // Group B: Compute-only buffers (starts at off_group_a) size_t off_group_b = off_group_a; @@ -454,6 +454,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( L->scratch_bytes[0] = scratch0_size; L->scratch_bytes[1] = scratch1_size; L->act_head_stride = 0; + L->src2_bytes = src2_size; off = off_group_a + hex_smax(group_b_size, group_c_size); } @@ -476,6 +477,7 @@ static inline void htp_mm_hvx_vtcm_layout_build( bool is_matmul_id, bool is_fused_nx ) { + (void)src1_row_size; size_t src0_sz = 0; size_t src1_sz = 0; size_t src2_sz = src2_row_size > 0 ? htp_mm_round_up(src2_row_size, 128) : 0; @@ -484,7 +486,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( const bool is_repack = (wtype == HTP_TYPE_Q4_0 || wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL || - wtype == HTP_TYPE_MXFP4); + wtype == HTP_TYPE_MXFP4 || wtype == HTP_TYPE_Q6_K || + wtype == HTP_TYPE_Q4_K); if (is_fused_nx) { const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); @@ -502,12 +505,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( weight_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128); } - size_t flat_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - size_t tiled_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - - size_t act_sz = (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) - ? hex_round_up(flat_act_row_size * src1_nrows, 128) - : hex_round_up(tiled_act_row_size * src1_nrows, 128); + size_t tiled_act_row_size = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + size_t act_sz = hex_round_up(tiled_act_row_size * src1_nrows, 128); src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch buffer src1_sz = act_sz; // quantized activation buffer @@ -516,8 +515,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( dst_sz = quant_scratch_size; } else if (is_matmul_id) { const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128); - const size_t src1_row_size_tiled = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) - : htp_mm_q8_0_tiled_row_size(ne10); + const size_t src1_row_size_tiled = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) + : htp_mm_q8_0_tiled_row_size(ne10); size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); src1_sz = htp_mm_round_up(src1_row_size_tiled * src1_nrows, 256); @@ -532,6 +531,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( src0_sz = src0_sz_per_thread * n_threads; dst_sz = htp_mm_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)) * n_threads; + src2_sz = 0; + src3_sz = 0; } else { const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128); const size_t dst_nrows = (src1_nrows > 1) ? 0 : 1; @@ -544,15 +545,6 @@ static inline void htp_mm_hvx_vtcm_layout_build( dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; break; } - case HTP_MM_KERNEL_HVX_F16_F32_DDR: - case HTP_MM_KERNEL_HVX_F16_F16_DDR: - case HTP_MM_KERNEL_HVX_F32_F32_DDR: - case HTP_MM_KERNEL_HVX_F32_F16_DDR: { - src0_sz = htp_mm_round_up(n_prefetch * src0_row_size, 256) * n_threads; - src1_sz = htp_mm_round_up(n_prefetch * src1_row_size, 256) * n_threads; - dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; - break; - } case HTP_MM_KERNEL_HVX_F32_F32_VTCM: { size_t f32_src1_row_size = htp_mm_round_up(ne10 * 4, 128); src1_sz = htp_mm_round_up(f32_src1_row_size * src1_nrows, 256); @@ -562,7 +554,7 @@ static inline void htp_mm_hvx_vtcm_layout_build( } case HTP_MM_KERNEL_HVX_QUANT_BLOCK: case HTP_MM_KERNEL_HVX_QUANT_ROW: { - size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); src1_sz = htp_mm_round_up(q_src1_row_size * src1_nrows, 256); @@ -583,28 +575,6 @@ static inline void htp_mm_hvx_vtcm_layout_build( dst_sz = dst_size_per_thread * n_threads; break; } - case HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT: { - size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - - src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); - src1_sz = htp_mm_round_up(q_src1_row_size * src1_nrows, 256); - - src0_sz = src0_sz * n_threads; - - if (is_repack) { - uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); - uint32_t n_k_tiles = ne10 / 32; - uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); - src0_sz = repacked_vtcm_size * n_threads; - } - - size_t quant_scratch_size_per_thread = htp_mm_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)); - size_t dst_slice_per_thread = dst_nrows > 0 ? htp_mm_round_up((dst_row_size + n_threads - 1) / n_threads, 128) : 0; - size_t dst_size_per_thread = (dst_slice_per_thread > quant_scratch_size_per_thread) ? dst_slice_per_thread : quant_scratch_size_per_thread; - dst_sz = dst_size_per_thread * n_threads; - break; - } default: break; } @@ -625,19 +595,99 @@ static inline void htp_mm_hvx_vtcm_layout_build( L->total_bytes = off; } +static inline bool htp_mm_hvx_solve_vtcm_params( + int kernel_type, + int wtype, + uint32_t ne10, + uint32_t src1_nrows, + uint32_t n_threads, + size_t dst_row_size, + size_t src0_row_size, + size_t src1_row_size, + size_t src2_row_size, + uint32_t n_prefetch, + size_t vtcm_budget, + struct htp_mm_hvx_vtcm_layout * L_out, + uint32_t * m_chunk_out +) { + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build( + &L, kernel_type, wtype, ne10, src1_nrows, n_threads, + dst_row_size, src0_row_size, src1_row_size, src2_row_size, n_prefetch, false, false + ); + + if (L.total_bytes <= vtcm_budget) { + *L_out = L; + *m_chunk_out = src1_nrows; + return true; + } + + const size_t fixed_bytes = L.src0_bytes + L.src2_bytes + L.dst_bytes; + if (vtcm_budget <= fixed_bytes) { + return false; + } + + const size_t avail_act = vtcm_budget - fixed_bytes; + size_t row_size = 0; + if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { + row_size = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) + ? htp_mm_q8_1_tiled_row_size(ne10) + : htp_mm_q8_0_tiled_row_size(ne10); + } else if (kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM) { + row_size = hex_round_up(ne10 * 2, 128); + } else { + row_size = hex_round_up(ne10 * 4, 128); + } + if (row_size == 0) { + return false; + } + + uint32_t m_chunk = (uint32_t) (avail_act / row_size); + if (m_chunk > 1) { + m_chunk &= ~1U; + } + if (m_chunk > src1_nrows) { + m_chunk = src1_nrows; + } + if (m_chunk < 1) { + return false; + } + + htp_mm_hvx_vtcm_layout_build( + &L, kernel_type, wtype, ne10, m_chunk, n_threads, + dst_row_size, src0_row_size, src1_row_size, src2_row_size, n_prefetch, false, false + ); + + while (m_chunk > 2 && L.total_bytes > vtcm_budget) { + m_chunk -= 2; + htp_mm_hvx_vtcm_layout_build( + &L, kernel_type, wtype, ne10, m_chunk, n_threads, + dst_row_size, src0_row_size, src1_row_size, src2_row_size, n_prefetch, false, false + ); + } + + if (L.total_bytes <= vtcm_budget) { + *L_out = L; + *m_chunk_out = m_chunk; + return true; + } + + return false; +} + static inline size_t htp_mm_hmx_get_2d_vtcm_size( - int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_threads, uint32_t aligned_tile_size + int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_threads, uint32_t aligned_tile_size, size_t src2_size ) { struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, false, pipeline, act_threads, aligned_tile_size); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, false, pipeline, act_threads, aligned_tile_size, src2_size); return L.total_bytes; } static inline size_t htp_mm_hmx_get_batched_vtcm_size( - int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool use_dma_activation, bool pipeline, uint32_t act_threads) { + int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool use_dma_activation, bool pipeline, uint32_t act_threads, size_t src2_size) { (void)pipeline; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_F16_BATCHED, wtype, k, mc, nc, group_size, use_dma_activation, false, act_threads, 0); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_F16_BATCHED, wtype, k, mc, nc, group_size, use_dma_activation, false, act_threads, 0, src2_size); return L.total_bytes; } @@ -650,6 +700,7 @@ static inline bool htp_mm_hmx_solve_batched_params( bool use_dma_activation, int n_threads, bool pipeline, + size_t src2_size, size_t vtcm_budget, size_t * m_chunk_out, size_t * n_chunk_out, @@ -664,7 +715,7 @@ static inline bool htp_mm_hmx_solve_batched_params( int act_threads = n_threads; while (act_threads >= 1) { - size_t group_overhead = htp_mm_hmx_get_batched_overhead(); + size_t group_overhead = htp_mm_hmx_get_batched_overhead() + (src2_size > 0 ? hex_align_up(src2_size, HTP_MM_HMX_TILE_SIZE) : 0); size_t group_size_per_n, group_size_per_m, group_size_per_mn; htp_mm_hmx_get_batched_chunk_costs(k, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn); @@ -675,7 +726,7 @@ static inline bool htp_mm_hmx_solve_batched_params( if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded, (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT, &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, use_dma_activation, pipeline, act_threads); + size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, use_dma_activation, pipeline, act_threads, src2_size); if (exact_size <= vtcm_budget) { size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate; if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { @@ -715,6 +766,7 @@ static inline bool htp_mm_hmx_solve_2d_params( bool pipeline, bool is_matmul_id, uint32_t aligned_tile_size, + size_t src2_size, size_t vtcm_budget, size_t * m_chunk_out, size_t * n_chunk_out, @@ -731,7 +783,7 @@ static inline bool htp_mm_hmx_solve_2d_params( int act_threads = n_threads; while (act_threads >= 1) { - size_t simple_2d_overhead = htp_mm_hmx_get_2d_overhead(pipeline, is_matmul_id); + size_t simple_2d_overhead = htp_mm_hmx_get_2d_overhead(pipeline, is_matmul_id) + (src2_size > 0 ? hex_align_up(src2_size, HTP_MM_HMX_TILE_SIZE) : 0); size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn; htp_mm_hmx_get_2d_chunk_costs(wtype, k, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn); @@ -742,7 +794,7 @@ static inline bool htp_mm_hmx_solve_2d_params( if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded, (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT, &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size); + size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size, src2_size); if (exact_size <= vtcm_budget) { size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate; if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.c b/ggml/src/ggml-hexagon/htp/pad-ops.c index aaa72b31590c..85f25a8eb760 100644 --- a/ggml/src/ggml-hexagon/htp/pad-ops.c +++ b/ggml/src/ggml-hexagon/htp/pad-ops.c @@ -7,13 +7,16 @@ #include <string.h> -#include "hex-dma.h" +#include "dma-queue.h" #include "hvx-utils.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" /* Circular wrap: maps any integer x into [0, n) */ static inline uint32_t wrap_around(int32_t x, uint32_t n) { @@ -48,6 +51,15 @@ static inline const uint8_t * pad_src_row_ptr(const struct htp_tensor * src, + (i3 - (uint32_t)lp3) * src->nb[3]; } +static inline dma_addr_t pad_src_row_data(const struct htp_tensor * src, + uint32_t i1, uint32_t i2, uint32_t i3, + int32_t lp1, int32_t lp2, int32_t lp3) { + return src->data + + (i1 - (uint32_t)lp1) * src->nb[1] + + (i2 - (uint32_t)lp2) * src->nb[2] + + (i3 - (uint32_t)lp3) * src->nb[3]; +} + /* Compute the DDR src row pointer for a circular row (wrap-around indexing) */ static inline const uint8_t * pad_circ_src_row_ptr(const struct htp_tensor * src, uint32_t i1, uint32_t i2, uint32_t i3, @@ -58,6 +70,15 @@ static inline const uint8_t * pad_circ_src_row_ptr(const struct htp_tensor * src + wrap_around((int32_t)i3 - lp3, src->ne[3]) * src->nb[3]; } +static inline dma_addr_t pad_circ_src_row_data(const struct htp_tensor * src, + uint32_t i1, uint32_t i2, uint32_t i3, + int32_t lp1, int32_t lp2, int32_t lp3) { + return src->data + + wrap_around((int32_t)i1 - lp1, src->ne[1]) * src->nb[1] + + wrap_around((int32_t)i2 - lp2, src->ne[2]) * src->nb[2] + + wrap_around((int32_t)i3 - lp3, src->ne[3]) * src->nb[3]; +} + struct htp_pad_context { struct htp_ops_context * octx; @@ -68,6 +89,7 @@ struct htp_pad_context { uint32_t nrows_per_thread; uint32_t total_dst_rows; + uint32_t row_start; size_t type_size; @@ -78,43 +100,43 @@ struct htp_pad_context { size_t dst_row_size_aligned; }; -#define htp_pad_preamble \ - const struct htp_tensor * src = octx->src[0]; \ - const struct htp_tensor * dst = octx->dst; \ - \ - const uint32_t ne00 = src->ne[0]; \ - const uint32_t nb00 = src->nb[0]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ - const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \ - const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \ - const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \ - const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \ - \ - const size_t type_size = pctx->type_size; \ - \ - const uint32_t row_start = pctx->nrows_per_thread * ith; \ - const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->total_dst_rows); - - -#define htp_pad_dma_preamble \ - const size_t src_row_size = pctx->src_row_size; \ - const size_t src_row_size_aligned = pctx->src_row_size_aligned; \ - const size_t dst_row_size = pctx->dst_row_size; \ - const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \ - \ +#define htp_pad_preamble \ + const struct htp_tensor * src = octx->src[0]; \ + const struct htp_tensor * dst = octx->dst; \ + \ + const uint32_t ne00 = src->ne[0]; \ + const uint32_t nb00 = src->nb[0]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ + \ + const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \ + const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \ + const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \ + const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \ + \ + const size_t type_size = pctx->type_size; \ + \ + const uint32_t row_start = pctx->row_start + pctx->nrows_per_thread * ith; \ + const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->row_start + pctx->total_dst_rows); + + +#define htp_pad_dma_preamble \ + const size_t src_row_size = pctx->src_row_size; \ + const size_t src_row_size_aligned = pctx->src_row_size_aligned; \ + const size_t dst_row_size = pctx->dst_row_size; \ + const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \ + \ uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \ uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \ \ - dma_queue * dma = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; // --------------------------------------------------------------------------- // HVX vectorized PAD kernel @@ -125,8 +147,8 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { uint32_t i1, i2, i3; @@ -165,18 +187,17 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); - FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } // --------------------------------------------------------------------------- -// HVX + DMA PAD kernel — aligned, double-buffered +// HVX + DMA PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) { @@ -185,9 +206,6 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void htp_pad_preamble; htp_pad_dma_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - // ----------------------------------------------------------------------- // Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the // double-buffer pipeline before the main loop begins. @@ -196,9 +214,9 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void uint8_t * src_spad_cur = src_spad_base + spad_idx * src_row_size_aligned; uint8_t * dst_spad_cur = dst_spad_base + spad_idx * dst_row_size_aligned; - dma_queue_push_vtcm_to_ddr(dma, - dma_make_ptr((uint8_t *)dst->data, dst_spad_cur), - dst_row_size, dst_row_size_aligned, 0); + dma_queue_push(dma_q, + dma_make_data(dst->data, dst_spad_cur), + dst_row_size, dst_row_size_aligned, dst_row_size, 0); uint32_t i1, i2, i3; pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3); @@ -207,35 +225,37 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void lp2, rp2, ne2, lp3, rp3, ne3); - const uint8_t * src_ptr = interior - ? pad_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3) : NULL; + const dma_addr_t src_data = interior + ? pad_src_row_data(src, i1, i2, i3, lp1, lp2, lp3) : src->data; // Interior row: real DMA (1 row) from DDR to VTCM. // Border row: null DMA (nrows=0) - dma_queue_push_ddr_to_vtcm(dma, - dma_make_ptr(src_spad_cur, - src_ptr ? src_ptr : (const uint8_t *)src_spad_cur), - src_row_size_aligned, src_row_size, src_ptr ? 1 : 0); + dma_queue_push(dma_q, + dma_make_data(src_spad_cur, src_data), + src_row_size_aligned, src_row_size, src_row_size, interior ? 1 : 0); } // ----------------------------------------------------------------------- // Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops, // push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = row_start; ir < row_end; ir++) { - uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src; - uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst; + uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i1, i2, i3; pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3); - uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; + const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; const int interior = pad_is_interior(i1, i2, i3, lp1, rp1, ne1, lp2, rp2, ne2, lp3, rp3, ne3); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); if (!interior) { hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0); } else { @@ -249,10 +269,11 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void hvx_copy_f32_ua(dst_interior, src_spad_cur, ne00); } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - dma_queue_push_vtcm_to_ddr(dma, - dma_make_ptr(dst_ptr, dst_spad_cur), - dst_row_size, dst_row_size_aligned, 1); + dma_queue_push(dma_q, + dma_make_data(dst_data, dst_spad_cur), + dst_row_size, dst_row_size_aligned, dst_row_size, 1); const uint32_t next_row = ir + 2; if (next_row < row_end) { @@ -262,26 +283,22 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void lp1, rp1, ne1, lp2, rp2, ne2, lp3, rp3, ne3); - const uint8_t * next_src_ptr = next_interior - ? pad_src_row_ptr(src, ni1, ni2, ni3, lp1, lp2, lp3) : NULL; + const dma_addr_t next_src_data = next_interior + ? pad_src_row_data(src, ni1, ni2, ni3, lp1, lp2, lp3) : src->data; - dma_queue_push_ddr_to_vtcm(dma, - dma_make_ptr(src_spad_cur, - next_src_ptr ? next_src_ptr : (const uint8_t *)src_spad_cur), - src_row_size_aligned, src_row_size, next_src_ptr ? 1 : 0); + dma_queue_push(dma_q, + dma_make_data(src_spad_cur, next_src_data), + src_row_size_aligned, src_row_size, src_row_size, next_interior ? 1 : 0); } } - dma_queue_flush(dma); + dma_queue_flush(dma_q); - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } // --------------------------------------------------------------------------- @@ -293,8 +310,8 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { uint32_t i1, i2, i3; @@ -344,18 +361,17 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); - FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } // --------------------------------------------------------------------------- -// HVX + DMA circular PAD kernel — aligned, double-buffered +// HVX + DMA circular PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) { @@ -364,9 +380,6 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i htp_pad_preamble; htp_pad_dma_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - // ----------------------------------------------------------------------- // Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the // double-buffer pipeline. Every row is a real src DMA (no null DMAs). @@ -375,30 +388,33 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i uint8_t * src_spad_cur = src_spad_base + spad_idx * src_row_size_aligned; uint8_t * dst_spad_cur = dst_spad_base + spad_idx * dst_row_size_aligned; - dma_queue_push_vtcm_to_ddr(dma, - dma_make_ptr((uint8_t *)dst->data, dst_spad_cur), - dst_row_size, dst_row_size_aligned, 0); + dma_queue_push(dma_q, + dma_make_data(dst->data, dst_spad_cur), + dst_row_size, dst_row_size_aligned, dst_row_size, 0); uint32_t pi1, pi2, pi3; pad_decompose_row(ir, ne1, ne2, &pi1, &pi2, &pi3); - dma_queue_push_ddr_to_vtcm(dma, - dma_make_ptr(src_spad_cur, pad_circ_src_row_ptr(src, pi1, pi2, pi3, lp1, lp2, lp3)), - src_row_size_aligned, src_row_size, 1); + const dma_addr_t src_data = pad_circ_src_row_data(src, pi1, pi2, pi3, lp1, lp2, lp3); + dma_queue_push(dma_q, + dma_make_data(src_spad_cur, src_data), + src_row_size_aligned, src_row_size, src_row_size, 1); } // ----------------------------------------------------------------------- // Main loop: pop completed DMAs, assemble circular row in VTCM with // aligned HVX ops, push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = row_start; ir < row_end; ir++) { - uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src; - uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst; + uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i1, i2, i3; pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3); - uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; - + const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); if (lp0 > 0) { uint8_t * dst_left = dst_spad_cur; const uint8_t * src_left = src_spad_cur + (size_t)(ne00 - (uint32_t)lp0) * type_size; @@ -430,32 +446,30 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i } } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - dma_queue_push_vtcm_to_ddr(dma, - dma_make_ptr(dst_ptr, dst_spad_cur), - dst_row_size, dst_row_size_aligned, 1); + dma_queue_push(dma_q, + dma_make_data(dst_data, dst_spad_cur), + dst_row_size, dst_row_size_aligned, dst_row_size, 1); const uint32_t next_row = ir + 2; if (next_row < row_end) { uint32_t nri1, nri2, nri3; pad_decompose_row(next_row, ne1, ne2, &nri1, &nri2, &nri3); - dma_queue_push_ddr_to_vtcm(dma, - dma_make_ptr(src_spad_cur, - pad_circ_src_row_ptr(src, nri1, nri2, nri3, lp1, lp2, lp3)), - src_row_size_aligned, src_row_size, 1); + const dma_addr_t next_src_data = pad_circ_src_row_data(src, nri1, nri2, nri3, lp1, lp2, lp3); + dma_queue_push(dma_q, + dma_make_data(src_spad_cur, next_src_data), + src_row_size_aligned, src_row_size, src_row_size, 1); } } - dma_queue_flush(dma); + dma_queue_flush(dma_q); - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } int op_pad(struct htp_ops_context * octx) { @@ -471,10 +485,6 @@ int op_pad(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; - } - const int32_t lp0 = octx->op_params[0]; const int32_t rp0 = octx->op_params[1]; const int32_t lp1 = octx->op_params[2]; @@ -489,21 +499,39 @@ int op_pad(struct htp_ops_context * octx) { const uint32_t ne00 = src0->ne[0]; const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_dst_rows > 0 ? total_dst_rows : 1); + const size_t dst_row_size = (size_t)ne0 * type_size; + + uint32_t row_start = 0; + uint32_t nrows = total_dst_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; const size_t src_row_size = (size_t)ne00 * type_size; - const size_t dst_row_size = (size_t)ne0 * type_size; const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN); const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); // Total VTCM needed: 2 buffers (ping+pong) for src and dst, per thread const size_t vtcm_needed = (size_t)n_threads * 2 * (src_row_size_aligned + dst_row_size_aligned); - const int use_dma = (src0->nb[0] == (uint32_t)type_size) && - (ne00 >= 512) && - (octx->ctx->vtcm_base != NULL) && + const int use_dma = (src0->nb[0] == (uint32_t)type_size) && (ne00 >= 512) && (octx->ctx->vtcm_size >= vtcm_needed); + if (!use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { + return HTP_STATUS_NO_SUPPORT; + } + if (use_dma) { octx->src0_spad.size_per_thread = 2 * src_row_size_aligned; octx->dst_spad.size_per_thread = 2 * dst_row_size_aligned; @@ -521,8 +549,9 @@ int op_pad(struct htp_ops_context * octx) { .lp1 = lp1, .rp1 = rp1, .lp2 = lp2, .rp2 = rp2, .lp3 = lp3, .rp3 = rp3, - .nrows_per_thread = (total_dst_rows + n_threads - 1) / n_threads, - .total_dst_rows = total_dst_rows, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_dst_rows = nrows, + .row_start = row_start, .type_size = type_size, .src_row_size = src_row_size, .src_row_size_aligned = src_row_size_aligned, @@ -537,11 +566,10 @@ int op_pad(struct htp_ops_context * octx) { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3); - if (circular && use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); } - else if (circular) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular, &pctx, n_threads); } - else if (use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_dma, &pctx, n_threads); } - else { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx, &pctx, n_threads); } + if (circular && use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); } + else if (circular) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads); } + else if (use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads); } + else { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads); } return HTP_STATUS_OK; } - diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.c b/ggml/src/ggml-hexagon/htp/repeat-ops.c index a6f2f0ed5f3f..2551be225b84 100644 --- a/ggml/src/ggml-hexagon/htp/repeat-ops.c +++ b/ggml/src/ggml-hexagon/htp/repeat-ops.c @@ -12,8 +12,10 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" struct htp_repeat_context { struct htp_ops_context * octx; @@ -25,6 +27,7 @@ struct htp_repeat_context { uint32_t nrows_per_thread; uint32_t total_dst_rows; // ne1 * ne2 * ne3 + uint32_t row_start; size_t type_size; }; @@ -62,11 +65,11 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat const size_t row_bytes = ne00 * rctx->type_size; - const uint32_t row_start = rctx->nrows_per_thread * ith; - const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->total_dst_rows); + const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows); - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { // Decompose flat dst row index into (i1, i2, i3) @@ -89,12 +92,12 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); - FARF(HIGH, "repeat %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "repeat %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } int op_repeat(struct htp_ops_context * octx) { @@ -119,21 +122,39 @@ int op_repeat(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_dst_rows); + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; + const size_t dst_row_size = dst->ne[0] * type_size; + + uint32_t row_start = 0; + uint32_t nrows = total_dst_rows; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { return HTP_STATUS_OK; } + const uint32_t n_threads = octx->n_threads; + struct htp_repeat_context rctx = { .octx = octx, .nr0 = dst->ne[0] / src0->ne[0], .nr1 = dst->ne[1] / src0->ne[1], .nr2 = dst->ne[2] / src0->ne[2], .nr3 = dst->ne[3] / src0->ne[3], - .nrows_per_thread = (total_dst_rows + n_threads - 1) / n_threads, - .total_dst_rows = total_dst_rows, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_dst_rows = nrows, + .row_start = row_start, .type_size = type_size, }; @@ -142,7 +163,7 @@ int op_repeat(struct htp_ops_context * octx) { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], rctx.nr0, rctx.nr1, rctx.nr2, rctx.nr3); - worker_pool_run_func(octx->ctx->worker_pool, repeat_job_per_thread, &rctx, n_threads); + work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.c b/ggml/src/ggml-hexagon/htp/roll-ops.c new file mode 100644 index 000000000000..9c373f56d3c9 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/roll-ops.c @@ -0,0 +1,316 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include <HAP_farf.h> +#include <HAP_perf.h> + +#include <string.h> + +#include "dma-queue.h" +#include "hvx-utils.h" + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "htp-ctx.h" +#include "hex-common.h" +#include "hex-profile.h" +#include "htp-ops.h" +#include "htp-tensor.h" + +struct htp_roll_context { + struct htp_ops_context * octx; + + uint32_t row_start; + uint32_t nrows; + uint32_t nrows_per_thread; + + struct fastdiv_values div_ne1; + struct fastdiv_values div_ne2_ne1; +}; + +static inline uint32_t htp_roll_wrap(int32_t i, uint32_t ne) { + if (i < 0) { + return (uint32_t) (i + (int32_t) ne); + } + if ((uint32_t) i >= ne) { + return (uint32_t) i - ne; + } + return (uint32_t) i; +} + +#define htp_roll_preamble \ + const struct htp_tensor * src0 = octx->src[0]; \ + const struct htp_tensor * dst = octx->dst; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ + \ + const int32_t s0 = octx->op_params[0]; \ + const int32_t s1 = octx->op_params[1]; \ + const int32_t s2 = octx->op_params[2]; \ + const int32_t s3 = octx->op_params[3]; \ + \ + const uint32_t i0_src0 = htp_roll_wrap(-s0, ne0); \ + const uint32_t n0 = ne0 - i0_src0; + +#define htp_roll_dma_preamble dma_queue * q = octx->ctx->dma[0]; + +static inline void roll_dma_push(dma_queue * q, + dma_addr_t dst, + dma_addr_t src, + uint32_t dst_stride, + uint32_t src_stride, + uint32_t bytes, + uint32_t nrows) { + if (bytes == 0 || nrows == 0) { + return; + } + + if (!dma_queue_push(q, dma_make_data(dst, src), dst_stride, src_stride, bytes, nrows)) { + dma_queue_flush(q); + dma_queue_push(q, dma_make_data(dst, src), + dst_stride, src_stride, bytes, nrows); + } +} + +static inline void roll_dma_push_rows(dma_queue * q, + const struct htp_tensor * dst, + const struct htp_tensor * src0, + uint32_t dst_row, + uint32_t src_row, + uint32_t nrows, + uint32_t row_size, + uint32_t i0_src0) { + const dma_addr_t dst_base = dst->data + (size_t) dst_row * row_size; + const dma_addr_t src_base = src0->data + (size_t) src_row * row_size; + const uint32_t n0 = src0->ne[0] - i0_src0; + + roll_dma_push(q, dst_base, src_base + (size_t) i0_src0 * sizeof(float), + row_size, row_size, n0 * sizeof(float), nrows); + roll_dma_push(q, dst_base + (size_t) n0 * sizeof(float), src_base, + row_size, row_size, i0_src0 * sizeof(float), nrows); +} + +// Same row-wrap split as roll_dma_push_rows, but addressed with explicit byte strides so it +// also works for a src0 that is row-contiguous only (e.g. a permuted view) rather than fully packed. +static inline void roll_dma_push_range(dma_queue * q, + dma_addr_t dst_row, + dma_addr_t src_row, + uint32_t dst_stride, + uint32_t src_stride, + uint32_t nrows, + uint32_t i0_src0, + uint32_t n0) { + roll_dma_push(q, dst_row, src_row + (size_t) i0_src0 * sizeof(float), + dst_stride, src_stride, n0 * sizeof(float), nrows); + roll_dma_push(q, dst_row + (size_t) n0 * sizeof(float), src_row, + dst_stride, src_stride, i0_src0 * sizeof(float), nrows); +} + +static int roll_dma_f32_contiguous(struct htp_ops_context * octx) { + htp_roll_preamble; + htp_roll_dma_preamble; + + const uint32_t row_size = ne0 * sizeof(float); + + if (s1 == 0 && s2 == 0 && s3 == 0) { + roll_dma_push_rows(q, dst, src0, 0, 0, ne1 * ne2 * ne3, row_size, i0_src0); + dma_queue_flush(q); + return HTP_STATUS_OK; + } + + if (s1 == 0) { + const uint32_t i2_src0 = htp_roll_wrap(-s2, ne2); + for (uint32_t i3 = 0; i3 < ne3; i3++) { + const uint32_t i03 = htp_roll_wrap((int32_t) i3 - s3, ne3); + const uint32_t dst_row0 = i3 * ne2 * ne1; + const uint32_t src_row0 = (i03 * ne2 + i2_src0) * ne1; + const uint32_t n2_first = ne2 - i2_src0; + + roll_dma_push_rows(q, dst, src0, dst_row0, src_row0, n2_first * ne1, + row_size, i0_src0); + roll_dma_push_rows(q, dst, src0, dst_row0 + n2_first * ne1, i03 * ne2 * ne1, + i2_src0 * ne1, row_size, i0_src0); + } + + dma_queue_flush(q); + return HTP_STATUS_OK; + } + + const uint32_t i1_src0 = htp_roll_wrap(-s1, ne1); + const uint32_t n1_first = ne1 - i1_src0; + + for (uint32_t i3 = 0; i3 < ne3; i3++) { + const uint32_t i03 = htp_roll_wrap((int32_t) i3 - s3, ne3); + for (uint32_t i2 = 0; i2 < ne2; i2++) { + const uint32_t i02 = htp_roll_wrap((int32_t) i2 - s2, ne2); + const uint32_t dst_row0 = (i3 * ne2 + i2) * ne1; + const uint32_t src_row0 = (i03 * ne2 + i02) * ne1; + + roll_dma_push_rows(q, dst, src0, dst_row0, src_row0 + i1_src0, + n1_first, row_size, i0_src0); + roll_dma_push_rows(q, dst, src0, dst_row0 + n1_first, src_row0, + i1_src0, row_size, i0_src0); + } + } + + dma_queue_flush(q); + return HTP_STATUS_OK; +} + +// DMA path for a row-contiguous but otherwise arbitrarily strided src0 (e.g. a permuted view). +// Same row-wrap split as above, one DMA push per (i2,i3), addressed via the real nb01/nb02/nb03 +// instead of assuming a packed layout. +static int roll_dma_f32_strided(struct htp_ops_context * octx) { + htp_roll_preamble; + htp_roll_dma_preamble; + + const uint32_t i1_src0 = htp_roll_wrap(-s1, ne1); + const uint32_t n1_first = ne1 - i1_src0; + + for (uint32_t i3 = 0; i3 < ne3; i3++) { + const uint32_t i03 = htp_roll_wrap((int32_t) i3 - s3, ne3); + for (uint32_t i2 = 0; i2 < ne2; i2++) { + const uint32_t i02 = htp_roll_wrap((int32_t) i2 - s2, ne2); + + const dma_addr_t dst_row0 = dst->data + (size_t) i2 * nb2 + (size_t) i3 * nb3; + const dma_addr_t src_row0 = src0->data + (size_t) i02 * nb02 + (size_t) i03 * nb03; + + roll_dma_push_range(q, dst_row0, src_row0 + (size_t) i1_src0 * nb01, + nb1, nb01, n1_first, i0_src0, n0); + roll_dma_push_range(q, dst_row0 + (size_t) n1_first * nb1, src_row0, + nb1, nb01, i1_src0, i0_src0, n0); + } + } + + dma_queue_flush(q); + return HTP_STATUS_OK; +} + +static void roll_thread_f32(unsigned int nth, unsigned int ith, void * data) { + struct htp_roll_context * rctx = (struct htp_roll_context *) data; + struct htp_ops_context * octx = rctx->octx; + + htp_roll_preamble; + + const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->nrows); + if (row_start >= row_end) { + return; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); + + for (uint32_t row = row_start; row < row_end; row++) { + const uint32_t i3 = fastdiv(row, &rctx->div_ne2_ne1); + const uint32_t rem = row - i3 * ne2 * ne1; + const uint32_t i2 = fastdiv(rem, &rctx->div_ne1); + const uint32_t i1 = rem - i2 * ne1; + + const uint32_t i01 = htp_roll_wrap((int32_t) i1 - s1, ne1); + const uint32_t i02 = htp_roll_wrap((int32_t) i2 - s2, ne2); + const uint32_t i03 = htp_roll_wrap((int32_t) i3 - s3, ne3); + + const uint8_t * src_row = (const uint8_t *) (uintptr_t) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + uint8_t * dst_row = (uint8_t *) (uintptr_t) dst->data + i1*nb1 + i2*nb2 + i3*nb3; + + hex_l2fetch(src_row + i0_src0 * sizeof(float), n0 * sizeof(float), ne0 * sizeof(float), 1); + hvx_copy_uu(dst_row, src_row + i0_src0 * sizeof(float), n0, sizeof(float)); + + if (i0_src0 != 0) { + hex_l2fetch(src_row, i0_src0 * sizeof(float), ne0 * sizeof(float), 1); + hvx_copy_uu(dst_row + n0 * sizeof(float), src_row, i0_src0, sizeof(float)); + } + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); + + FARF(HIGH, "roll %d/%d: (%ux%ux%ux%u) rows %u:%u shift=(%d,%d,%d,%d)\n", + ith, nth, ne0, ne1, ne2, ne3, + row_start, row_end, s0, s1, s2, s3); +} + +int execute_op_roll_f32(struct htp_ops_context * octx) { + htp_roll_preamble; + + if (src0->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) { + FARF(ERROR, "roll: unsupported type %u -> %u\n", src0->type, dst->type); + return HTP_STATUS_NO_SUPPORT; + } + + if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + FARF(ERROR, "roll: unsupported nb0 %u -> %u\n", src0->nb[0], dst->nb[0]); + return HTP_STATUS_NO_SUPPORT; + } + + if (src0->ne[0] != ne0 || src0->ne[1] != ne1 || + src0->ne[2] != ne2 || src0->ne[3] != ne3) { + FARF(ERROR, "roll: shape mismatch\n"); + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t total_rows = ne1 * ne2 * ne3; + const size_t dst_row_size = ne0 * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + if (octx->ctx->mdev.count <= 1) { + if (htp_tensor_is_contiguous(src0, sizeof(float)) && htp_tensor_is_contiguous(dst, sizeof(float))) { + return roll_dma_f32_contiguous(octx); + } + return roll_dma_f32_strided(octx); + } + + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + const uint32_t n_threads = octx->n_threads; + struct htp_roll_context rctx = { + .octx = octx, + .row_start = row_start, + .nrows = nrows, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .div_ne1 = init_fastdiv_values(dst->ne[1]), + .div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]), + }; + + work_queue_run(octx->ctx->work_queue, roll_thread_f32, &rctx, n_threads); + + return HTP_STATUS_OK; +} + +int op_roll(struct htp_ops_context * octx) { + switch (octx->src[0]->type) { + case HTP_TYPE_F32: + return execute_op_roll_f32(octx); + + default: + return HTP_STATUS_NO_SUPPORT; + } +} diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 6c689824934f..f6b4d383ce4c 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -9,7 +9,7 @@ #include <string.h> #include <stdlib.h> -#include "hex-dma.h" +#include "dma-queue.h" #include "hvx-utils.h" #include "hex-fastdiv.h" @@ -17,8 +17,8 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" #include "htp-tensor.h" +#include "rope-ops.h" // Redefined the rope type constants as we can't include ggml.h #define HTP_ROPE_TYPE_NORMAL 0 @@ -27,9 +27,6 @@ #define HTP_ROPE_TYPE_VISION 24 #define HTP_ROPE_TYPE_IMROPE 40 -#define HTP_ROPE_SPAD_NROWS 16 -#define HTP_ROPE_SPAD_BLOCK (HTP_ROPE_SPAD_NROWS/2) - #define htp_rope_preamble \ const uint32_t ne00 = src0->ne[0]; \ const uint32_t ne01 = src0->ne[1]; \ @@ -65,26 +62,31 @@ struct htp_rope_context { float beta_fast; float beta_slow; float theta_scale; + float theta_scale_32; + float theta_powers[32]; float corr_dims[2]; uint32_t src0_nrows_per_thread; - size_t spad_stride; struct htp_ops_context * octx; + uint8_t * vtcm_base; + size_t spad_per_thread; + size_t theta_cache_offset; + size_t src0_row_size; size_t src0_row_stride; size_t dst_row_size; size_t dst_row_stride; size_t src0_row_size_aligned; - size_t dst_row_size_aligned; - size_t theta_cache_offset; uint32_t src0_nrows; + uint32_t row_start; + uint32_t nrows; struct fastdiv_values div_ne2_ne1; struct fastdiv_values div_ne1; - uint64_t t_start; + const float * freq_factors; }; static float rope_yarn_ramp(const float low, const float high, const int i0) { @@ -112,94 +114,80 @@ static inline void rope_yarn_one(float theta, float freq_scale, float * corr_dim mscale_final *= 1.0f + 0.1f * logf(1.0f / freq_scale); } - cache[i0 + 0] = cosf(theta_final) * mscale_final; - cache[i0 + 1] = sinf(theta_final) * mscale_final; + const uint32_t b = i0 / 64; + const uint32_t k = (i0 % 64) / 2; + cache[b * 64 + k] = cosf(theta_final) * mscale_final; + cache[b * 64 + 32 + k] = sinf(theta_final) * mscale_final; +} + +// 32 thetas -> 32 deinterleaved pairs [cos[32] | sin[32]] at cache[i0]. +static inline void rope_cache_hvx_32(float * cache, uint32_t i0, + HVX_Vector v_theta, + const float * freq_factors, + HVX_Vector v_freq_scale, + HVX_Vector v_mscale) { + if (freq_factors) { + HVX_Vector v_ff = hvx_vmemu(freq_factors + i0 / 2); + v_theta = hvx_vec_mul_f32_f32(v_theta, hvx_vec_inverse_f32(v_ff)); + } + + HVX_Vector v_theta_final = hvx_vec_mul_f32_f32(v_theta, v_freq_scale); + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(v_theta_final, &vcos, &vsin); + vcos = hvx_vec_mul_f32_f32(vcos, v_mscale); + vsin = hvx_vec_mul_f32_f32(vsin, v_mscale); + + if (((uintptr_t) (cache + i0)) % 128 == 0) { + hvx_vmem(cache + i0 + 0) = vcos; + hvx_vmem(cache + i0 + 32) = vsin; + } else { + hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), vcos); + hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), vsin); + } } static __attribute__((noinline)) void rope_cache_init(const float theta_base, const float freq_scale, const float * freq_factors, float * corr_dims, - const uint32_t ne0, + const uint32_t n_cache, const float ext_factor, const float mscale, float * cache, - const float theta_scale) { + const float theta_scale, + const float * theta_powers, + const float theta_scale_32) { // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py -#if __HVX_ARCH__ >= 79 - const bool is_v79_or_newer = true; -#else - const bool is_v79_or_newer = false; -#endif - - if (is_v79_or_newer && ext_factor == 0.0f) { + if (ext_factor == 0.0f) { // Fast path: fully vectorized // We process 32 pairs (64 elements) per iteration. - const uint32_t n_blocks = ne0 / 64; - - // Initialize theta scale powers: [1.0f, theta_scale, theta_scale^2, ..., theta_scale^31] - float __attribute__((aligned(128))) theta_powers[32]; - theta_powers[0] = 1.0f; - for (int j = 1; j < 32; j++) { - theta_powers[j] = theta_powers[j - 1] * theta_scale; - } - HVX_Vector v_theta_powers = hvx_vmem(theta_powers); + const uint32_t n_blocks = n_cache / 64; + HVX_Vector v_theta_powers = hvx_vmemu(theta_powers); HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); - // Base theta starts at theta_base float theta_block = theta_base; - // The scale factor for the next block is theta_scale^32 - float theta_scale_32 = 1.0f; - for (int j = 0; j < 32; j++) { - theta_scale_32 *= theta_scale; - } for (uint32_t b = 0; b < n_blocks; b++) { uint32_t i0 = b * 64; HVX_Vector v_theta_base = hvx_vec_splat_f32(theta_block); HVX_Vector v_theta = hvx_vec_mul_f32_f32(v_theta_base, v_theta_powers); - - if (freq_factors) { - // Load 32 elements of freq_factors - HVX_Vector v_ff = hvx_vmemu(freq_factors + i0 / 2); - HVX_Vector v_inv_ff = hvx_vec_inverse_f32(v_ff); - v_theta = hvx_vec_mul_f32_f32(v_theta, v_inv_ff); - } - - HVX_Vector v_theta_final = hvx_vec_mul_f32_f32(v_theta, v_freq_scale); - - HVX_Vector vcos = hvx_vec_cos_f32(v_theta_final); - HVX_Vector vsin = hvx_vec_sin_f32(v_theta_final); - - vcos = hvx_vec_mul_f32_f32(vcos, v_mscale); - vsin = hvx_vec_mul_f32_f32(vsin, v_mscale); - - HVX_VectorPair vstore = Q6_W_vshuff_VVR(vsin, vcos, -4); - - if (((uintptr_t)cache) % 128 == 0) { - hvx_vmem(cache + i0 + 0) = Q6_V_lo_W(vstore); - hvx_vmem(cache + i0 + 32) = Q6_V_hi_W(vstore); - } else { - hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), Q6_V_lo_W(vstore)); - hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), Q6_V_hi_W(vstore)); - } - + rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale); theta_block *= theta_scale_32; } // Leftovers float theta = theta_block; - for (uint32_t i0 = n_blocks * 64; i0 < ne0; i0 += 2) { + for (uint32_t i0 = n_blocks * 64; i0 < n_cache; i0 += 2) { const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); theta *= theta_scale; } } else { - // Fallback to original scalar loop float theta = theta_base; - for (uint32_t i0 = 0; i0 < ne0; i0 += 2) { + for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) { const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); theta *= theta_scale; @@ -207,6 +195,72 @@ static __attribute__((noinline)) void rope_cache_init(const float theta_base, } } +static inline float mrope_pick_theta(float theta_t, float theta_h, float theta_w, float theta_e, + int sector, const int32_t sections[4], int sec_w, int sec_e, + bool is_imrope) { + if (is_imrope) { + if (sector % 3 == 0 && sector < 3 * sections[0]) { return theta_t; } + else if (sector % 3 == 1 && sector < 3 * sections[1]) { return theta_h; } + else if (sector % 3 == 2 && sector < 3 * sections[2]) { return theta_w; } + else { return theta_e; } + } + if (sector < sections[0]) { return theta_t; } + else if (sector < sec_w) { return theta_h; } + else if (sector < sec_e) { return theta_w; } + else { return theta_e; } +} + +// lane j is 1 when (j % 3) == rem +static const float __attribute__((aligned(128))) mrope_mod3_eq0[32] = { + 1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0 +}; +static const float __attribute__((aligned(128))) mrope_mod3_eq1[32] = { + 0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1 +}; +static const float __attribute__((aligned(128))) mrope_mod3_eq2[32] = { + 0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0 +}; + +static const float __attribute__((aligned(128))) mrope_k_ramp[32] = { + 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, + 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +}; + +static inline HVX_VectorPred mrope_mask_eq1(const float * m) { + return Q6_Q_vcmp_gt_VsfVsf(hvx_vmemu(m), Q6_V_vzero()); +} + +// IMROPE without wrap: theta[k] = pos[k % 3] * scale^k +static inline HVX_Vector mrope_thetas_imrope_mod3(float pos_t, float pos_h, float pos_w, + uint32_t k0, HVX_Vector v_powers, float scale_block) { + const int r = (int) (k0 % 3); + const float * mt = (r == 0) ? mrope_mod3_eq0 : (r == 1) ? mrope_mod3_eq2 : mrope_mod3_eq1; + const float * mh = (r == 0) ? mrope_mod3_eq1 : (r == 1) ? mrope_mod3_eq0 : mrope_mod3_eq2; + + HVX_Vector v = hvx_vec_splat_f32(pos_w); + v = Q6_V_vmux_QVV(mrope_mask_eq1(mh), hvx_vec_splat_f32(pos_h), v); + v = Q6_V_vmux_QVV(mrope_mask_eq1(mt), hvx_vec_splat_f32(pos_t), v); + v = hvx_vec_mul_f32_f32(v, v_powers); + return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block)); +} + +// Contiguous MROPE without wrap: theta[k] = pos[section(k)] * scale^k +static inline HVX_Vector mrope_thetas_contig(float pos_t, float pos_h, float pos_w, float pos_e, + uint32_t k0, int s0, int sec_w, int sec_e, + HVX_Vector v_powers, float scale_block) { + HVX_Vector v_k = hvx_vec_add_f32_f32(hvx_vec_splat_f32((float) k0), hvx_vmemu(mrope_k_ramp)); + HVX_VectorPred lt_s0 = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) s0), v_k); + HVX_VectorPred lt_sw = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_w), v_k); + HVX_VectorPred lt_se = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_e), v_k); + + HVX_Vector v = hvx_vec_splat_f32(pos_e); + v = Q6_V_vmux_QVV(lt_se, hvx_vec_splat_f32(pos_w), v); + v = Q6_V_vmux_QVV(lt_sw, hvx_vec_splat_f32(pos_h), v); + v = Q6_V_vmux_QVV(lt_s0, hvx_vec_splat_f32(pos_t), v); + v = hvx_vec_mul_f32_f32(v, v_powers); + return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block)); +} + // pos_t/h/w/e: the four position ids for this sequence step (t=time, h=height, w=width, e=extra). // sections[4]: number of head dims assigned to each position component. static __attribute__((noinline)) void mrope_cache_init(const float pos_t, @@ -219,23 +273,71 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t, const float freq_scale, const float * freq_factors, float * corr_dims, - const uint32_t ne0, + const uint32_t n_cache, const float ext_factor, const float mscale, float * cache, - const float theta_scale) { + const float theta_scale, + const float * theta_powers, + const float theta_scale_32) { const int sect_dims = sections[0] + sections[1] + sections[2] + sections[3]; const int sec_w = sections[0] + sections[1]; const int sec_e = sec_w + sections[2]; + const uint32_t n_pairs = n_cache / 2; + + const bool no_wrap = (sect_dims > 0) && (n_pairs <= (uint32_t) sect_dims); + const bool imrope_mod3 = is_imrope && !indep_sects && no_wrap + && sections[0] > 0 && sections[1] > 0 && sections[2] > 0 + && n_pairs <= (uint32_t) (3 * sections[0]) + && n_pairs <= (uint32_t) (3 * sections[1]) + && n_pairs <= (uint32_t) (3 * sections[2]); + const bool contig = !is_imrope && !indep_sects && no_wrap; + + if (ext_factor == 0.0f && (imrope_mod3 || contig)) { + HVX_Vector v_powers = hvx_vmemu(theta_powers); + HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); + HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); + float scale_block = 1.0f; + const uint32_t n_blocks = n_cache / 64; + + for (uint32_t b = 0; b < n_blocks; b++) { + const uint32_t i0 = b * 64; + const uint32_t k0 = b * 32; + HVX_Vector v_theta = imrope_mod3 + ? mrope_thetas_imrope_mod3(pos_t, pos_h, pos_w, k0, v_powers, scale_block) + : mrope_thetas_contig(pos_t, pos_h, pos_w, pos_e, k0, sections[0], sec_w, sec_e, + v_powers, scale_block); + rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale); + scale_block *= theta_scale_32; + } + + float theta_k = scale_block; + for (uint32_t k = n_blocks * 32; k < n_pairs; k++) { + const uint32_t i0 = 2 * k; + const float pos = mrope_pick_theta(pos_t, pos_h, pos_w, pos_e, + (int) k, sections, sec_w, sec_e, is_imrope); + const float ff = freq_factors ? freq_factors[k] : 1.0f; + rope_yarn_one(pos * theta_k / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); + theta_k *= theta_scale; + } + return; + } float theta_t = pos_t; float theta_h = pos_h; float theta_w = pos_w; float theta_e = pos_e; - for (uint32_t i0 = 0; i0 < ne0; i0 += 2) { - const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; - const int sector = (i0 / 2) % sect_dims; + const bool use_hvx = (ext_factor == 0.0f); + float __attribute__((aligned(128))) thetas[32]; + uint32_t n_thetas = 0; + uint32_t block_i0 = 0; + + HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); + HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); + + for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) { + const int sector = (i0 / 2) % sect_dims; if (indep_sects) { // Reset theta when crossing into a new section. @@ -245,28 +347,34 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t, else if (sector == sec_e) { theta_e = pos_e; } } - float theta; - if (is_imrope) { - // Interleaved: sector mod 3 selects component - if (sector % 3 == 0 && sector < 3 * sections[0]) { theta = theta_t; } - else if (sector % 3 == 1 && sector < 3 * sections[1]) { theta = theta_h; } - else if (sector % 3 == 2 && sector < 3 * sections[2]) { theta = theta_w; } - else { theta = theta_e; } + const float theta = mrope_pick_theta(theta_t, theta_h, theta_w, theta_e, + sector, sections, sec_w, sec_e, is_imrope); + + if (use_hvx) { + if (n_thetas == 0) { + block_i0 = i0; + } + thetas[n_thetas++] = theta; + if (n_thetas == 32) { + rope_cache_hvx_32(cache, block_i0, hvx_vmemu(thetas), freq_factors, v_freq_scale, v_mscale); + n_thetas = 0; + } } else { - // Contiguous sections - if (sector < sections[0]) { theta = theta_t; } - else if (sector < sec_w) { theta = theta_h; } - else if (sector < sec_e) { theta = theta_w; } - else { theta = theta_e; } + const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; + rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); } - rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); - theta_t *= theta_scale; theta_h *= theta_scale; theta_w *= theta_scale; theta_e *= theta_scale; } + + for (uint32_t k = 0; k < n_thetas; k++) { + const uint32_t i0 = block_i0 + 2 * k; + const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; + rope_yarn_one(thetas[k] / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); + } } #define M_PI 3.1415926535897932384626433 @@ -283,52 +391,54 @@ static void rope_corr_dims(int n_dims, dims[1] = MIN(n_dims - 1, end); } +static inline void hvx_rope_neox_mul(HVX_Vector v0, HVX_Vector v1, HVX_Vector vcos, HVX_Vector vsin, + HVX_Vector * o0, HVX_Vector * o1) { + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, vsin); + *o0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s)); + *o1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c)); +} + +// theta_cache full 32-pair blocks are deinterleaved [cos | sin]. static inline void hvx_rope_neox_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) { const uint32_t he = ne / 2; const uint32_t nvec = he / 32; const uint32_t nloe = he % 32; - for (uint32_t i = 0; i < nvec; i++) { - HVX_Vector v0 = ((const HVX_Vector *) src0)[i]; - HVX_Vector v1 = hvx_vmemu(src0 + he + i * 32); - - HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[i * 2 + 0]; - HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[i * 2 + 1]; - - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); - - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin)); - - HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); - HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); - - ((HVX_Vector *) dst)[i] = Q6_Vsf_equals_Vqf32(v4); - hvx_vmemu(dst + he + i * 32) = Q6_Vsf_equals_Vqf32(v5); + if (nloe == 0) { + const HVX_Vector * vs = (const HVX_Vector *) src0; + const HVX_Vector * vt = (const HVX_Vector *) theta_cache; + HVX_Vector * vd = (HVX_Vector *) dst; + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector o0, o1; + hvx_rope_neox_mul(vs[i], vs[nvec + i], vt[i * 2 + 0], vt[i * 2 + 1], &o0, &o1); + vd[i] = o0; + vd[nvec + i] = o1; + } + return; } - if (nloe > 0) { - HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32); - HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32); - - HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[nvec * 2 + 0]; - HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[nvec * 2 + 1]; - - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); - - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin)); - - HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); - HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); - - hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v4)); - hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v5)); + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector o0, o1; + hvx_rope_neox_mul(((const HVX_Vector *) src0)[i], + hvx_vmemu(src0 + he + i * 32), + ((const HVX_Vector *) theta_cache)[i * 2 + 0], + ((const HVX_Vector *) theta_cache)[i * 2 + 1], + &o0, &o1); + ((HVX_Vector *) dst)[i] = o0; + hvx_vmemu(dst + he + i * 32) = o1; } + + HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32); + HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); + HVX_Vector o0, o1; + hvx_rope_neox_mul(v0, v1, vcos, vsin, &o0, &o1); + hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), o0); + hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), o1); } static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) { @@ -339,16 +449,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict HVX_Vector v0 = ((const HVX_Vector *) src0)[i * 2 + 0]; HVX_Vector v1 = ((const HVX_Vector *) src0)[i * 2 + 1]; - HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[i * 2 + 0]; - HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[i * 2 + 1]; + HVX_Vector vcos = ((const HVX_Vector *) theta_cache)[i * 2 + 0]; + HVX_Vector vsin = ((const HVX_Vector *) theta_cache)[i * 2 + 1]; - HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin)); + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); @@ -362,15 +471,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict if (nloe > 0) { if (nloe <= 32) { HVX_Vector v0 = hvx_vmemu(src0 + nvec * 64); - HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); - HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(Q6_V_vzero(), v0, -4); - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(Q6_V_vzero(), v2, -4); + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(Q6_V_vzero(), v0, -4); - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin)); + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); @@ -382,16 +491,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict HVX_Vector v0 = hvx_vmemu(src0 + nvec * 64); HVX_Vector v1 = hvx_vmemu(src0 + nvec * 64 + 32); - HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64); - HVX_Vector v3 = hvx_vmemu(theta_cache + nvec * 64 + 32); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); - HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin)); + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); @@ -404,54 +512,23 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict } } -static void inline rope_basic_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { - const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) - #pragma unroll(4) - for (uint32_t i = 0; i < nr; i++) { - float * d = (float *) (dst + i * rctx->dst_row_size_aligned); - float * s = (float *) (src + i * rctx->src0_row_size_aligned); - - hvx_rope_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache); - - // fill the remain channels with data from src tensor - if (n_offs > 0) { - hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs); - } - if (n_offs + rctx->n_dims < ne0) { - hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims); - } - } -} - -static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { - const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) +static void inline rope_basic_f32_inplace(struct htp_rope_context * rctx, uint8_t * src, + uint32_t nr, const float * restrict theta_cache) { + const uint32_t n_offs = rctx->n_offs; #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { - float * d = (float *) (dst + i * rctx->dst_row_size_aligned); float * s = (float *) (src + i * rctx->src0_row_size_aligned); - - hvx_rope_neox_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache); - - // fill the remain channels with data from src tensor - if (n_offs > 0) { - hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs); - } - if (n_offs + rctx->n_dims < ne0) { - hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims); - } + hvx_rope_f32_aa(s + n_offs, s + n_offs, rctx->n_dims, theta_cache); } } -static void inline rope_vision_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { +static void inline rope_neox_f32_inplace(struct htp_rope_context * rctx, uint8_t * src, + uint32_t nr, uint32_t ne, const float * restrict theta_cache) { + const uint32_t n_offs = rctx->n_offs; #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { - float * d = (float *) (dst + i * rctx->dst_row_size_aligned); float * s = (float *) (src + i * rctx->src0_row_size_aligned); - - hvx_rope_neox_f32_aa(d, s, ne0, theta_cache); + hvx_rope_neox_f32_aa(s + n_offs, s + n_offs, ne, theta_cache); } } @@ -466,33 +543,31 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { htp_rope_preamble; - const uint32_t src0_nrows = rctx->src0_nrows; + const uint32_t src0_nrows = rctx->nrows; const uint32_t src0_nrows_per_thread = rctx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, rctx->row_start + src0_nrows); // no work for this thread if (src0_start_row >= src0_end_row) { return; } - uint64_t tt = HAP_perf_get_qtimer_count(); - const int32_t mode = rctx->mode; // MROPE, IMROPE and VISION use NEOX-style pairing for the rotation const bool is_neox = (mode & HTP_ROPE_TYPE_NEOX) || (mode & HTP_ROPE_TYPE_MROPE); const bool is_vision = (mode == HTP_ROPE_TYPE_VISION); // VTCM setup - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); + uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_thread); float * theta_cache = (float *) (src0_spad_base); src0_spad_base = src0_spad_base + rctx->theta_cache_offset; - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); - dma_queue * dma_queue = octx->ctx->dma[ith]; - const int32_t * pos = (const int32_t *) src1->data; - const float * freq_factors = src2 ? (const float *) src2->data : NULL; + dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + const int32_t * pos = (const int32_t *) (uintptr_t) src1->data; + const float * freq_factors = rctx->freq_factors; const uint32_t i3_start = fastdiv(src0_start_row, &rctx->div_ne2_ne1); const uint32_t rem = fastmodulo(src0_start_row, ne2 * ne1, &rctx->div_ne2_ne1); @@ -501,6 +576,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { uint32_t ir = src0_start_row; uint32_t prev_i2 = (uint32_t) -1; + uint32_t cur_slot = 0; for (uint32_t i3 = i3_start; i3 < ne3; i3++) { // batch const uint32_t i2_init = (i3 == i3_start) ? i2_start : 0; @@ -513,35 +589,30 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { const uint32_t nrows = MIN(src0_end_row - ir, ne1 - i1); // Depth before prefetch - uint32_t dma_depth = dma_queue_depth(dma_queue); - - // FARF(HIGH, "rope-block %u: ir %u n-rows %u dma-depth %u : usec %u", ith, ir, nrows, dma_depth, - // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start)); - - // Prefetch loop - for (uint32_t pnr = 0, pr = 0; pr < nrows && pr < HTP_ROPE_SPAD_NROWS; pr += pnr) { - pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK); + const uint32_t dma_depth = dma_queue_depth(dma_q); - uint32_t pi1 = i1 + pr; - uint32_t pir = ir + pr; + // Prefetch up to 2 blocks + const uint32_t p_nrows = MIN(nrows, 2 * HTP_ROPE_SPAD_BLOCK); + for (uint32_t pr = 0; pr < p_nrows; pr += HTP_ROPE_SPAD_BLOCK) { + const uint32_t pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK); + const uint32_t slot = (cur_slot + pr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + uint8_t * spad_slot = rope_spad_slot(src0_spad_base, slot, rctx->src0_row_size_aligned); + const dma_addr_t src0_data = src0->data + i3 * nb03 + i2 * nb02 + (i1 + pr) * nb01; - // Dummy DMA transaction for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr((void *) dst->data, dst_spad_base + pr * rctx->dst_row_size_aligned), 0, 0, 0); + // Dummy DMA transaction for sequencing (interleaving wr, rd, wr, rd, ...) + dma_queue_push(dma_q, dma_make_data(dst->data, spad_slot), 0, 0, 0, 0); - const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01; - uint8_t * src_spad = src0_spad_base + pr * rctx->src0_row_size_aligned; - - // Copy only the row payload while striding the DDR source - dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr), + dma_queue_push(dma_q, dma_make_data(spad_slot, src0_data), rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); - - // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr); } // Update theta cache if (i2 != prev_i2) { prev_i2 = i2; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, i2); + // VISION rotates the full row; other modes only rotate n_dims. + const uint32_t n_cache = is_vision ? ne0 : (uint32_t) rctx->n_dims; const bool is_mrope = (rctx->mode & HTP_ROPE_TYPE_MROPE) != 0; if (is_mrope) { // src1 holds four position arrays stacked along ne0: @@ -554,66 +625,71 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { (float) pos[i2 + ne2 * 3], rctx->sections, is_imrope, is_vision, rctx->freq_scale, freq_factors, rctx->corr_dims, - ne0, rctx->ext_factor, rctx->attn_factor, - theta_cache, rctx->theta_scale); + n_cache, rctx->ext_factor, rctx->attn_factor, + theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32); } else { rope_cache_init(pos[i2], rctx->freq_scale, freq_factors, rctx->corr_dims, - ne0, rctx->ext_factor, rctx->attn_factor, - theta_cache, rctx->theta_scale); + n_cache, rctx->ext_factor, rctx->attn_factor, + theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, i2); } // Skip output DMA transactions from prev block (if any) - // No need to wait for those here since we're explicitly waiting for the latest prefecthes below. - for (uint32_t d=0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_queue); } + for (uint32_t d = 0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_q); } // Compute loop - for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr, ir += cnr, i1 += cnr) { - // Number of rows to compute - cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK); + const uint32_t ne = is_vision ? ne0 : rctx->n_dims; + const uint32_t base_i1 = i1; + const uint32_t base_ir = ir; - uint8_t * dst_spad = (uint8_t *) dma_queue_pop(dma_queue).src; - uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_queue).dst; + for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr) { + cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK); + const uint32_t slot = (cur_slot + cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + const uint32_t cur_ir = base_ir + cr; + const uint32_t cur_i1 = base_i1 + cr; - // FARF(HIGH, "rope-compute %u: ir %u i1 %u i2 %u i3 %u src-spad %p cnr %u : usec %u", ith, ir, i1, i2, i3, src_spad, cnr, - // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start)); + dma_queue_pop(dma_q); + uint8_t * cur_spad = (uint8_t *) dma_queue_pop(dma_q).dst; - if (is_vision) { - rope_vision_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); - } else if (is_neox) { - rope_neox_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir); + if (is_neox || is_vision) { + rope_neox_f32_inplace(rctx, cur_spad, cnr, ne, theta_cache); } else { - rope_basic_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); + rope_basic_f32_inplace(rctx, cur_spad, cnr, theta_cache); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir); - uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1; + const dma_addr_t dst_data = dst->data + i3 * nb3 + i2 * nb2 + cur_i1 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_data, cur_spad), + rctx->dst_row_stride, rctx->src0_row_size_aligned, rctx->dst_row_size, cnr); - // Write only the row payload while striding the DDR dst - dma_queue_push(dma_queue, dma_make_ptr(dst_addr, dst_spad), - rctx->dst_row_stride, rctx->dst_row_size_aligned, rctx->dst_row_size, cnr); + // Prefetch 2 blocks ahead into the slot just freed + if ((cr + 2 * HTP_ROPE_SPAD_BLOCK) < nrows) { + const uint32_t p_cr = cr + 2 * HTP_ROPE_SPAD_BLOCK; + const uint32_t pnr = MIN(nrows - p_cr, HTP_ROPE_SPAD_BLOCK); + const uint32_t p_slot = (cur_slot + p_cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + uint8_t * p_spad = rope_spad_slot(src0_spad_base, p_slot, rctx->src0_row_size_aligned); + const dma_addr_t p_src0_data = src0->data + i3 * nb03 + i2 * nb02 + (base_i1 + p_cr) * nb01; - // Prefetch more rows (if any) - if ((cr + HTP_ROPE_SPAD_NROWS) < nrows) { - uint32_t pnr = MIN(nrows - (cr + HTP_ROPE_SPAD_NROWS), HTP_ROPE_SPAD_BLOCK); - uint32_t pi1 = i1 + HTP_ROPE_SPAD_NROWS; - uint32_t pir = ir + HTP_ROPE_SPAD_NROWS; - - const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01; - dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr), + dma_queue_push(dma_q, dma_make_data(p_spad, p_src0_data), rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); - - // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr); } } + + const uint32_t n_chunks = (nrows + HTP_ROPE_SPAD_BLOCK - 1) / HTP_ROPE_SPAD_BLOCK; + cur_slot = (cur_slot + n_chunks) % HTP_ROPE_SPAD_NSLOTS; + + ir += nrows; + i1 += nrows; } } } done: - dma_queue_flush(dma_queue); - tt = HAP_perf_get_qtimer_count() - tt; + dma_queue_flush(dma_q); - FARF(HIGH, "rope-f32: %d/%d: (%u:%u) usec %u\n", ith, nth, src0_start_row, src0_end_row, (unsigned) HAP_perf_qtimer_count_to_us(tt)); + FARF(HIGH, "rope-f32: %d/%d: (%u:%u)\n", ith, nth, src0_start_row, src0_end_row); } static int execute_op_rope_f32(struct htp_ops_context * octx) { @@ -624,8 +700,6 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { const struct htp_tensor * src2 = octx->src[2]; const struct htp_tensor * dst = octx->dst; - const char * op_type = "rope-f32"; - switch (octx->op) { case HTP_OP_ROPE: break; @@ -635,48 +709,60 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t ne0 = dst->ne[0]; - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); + const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params; + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + assert(octx->ctx->vtcm_size >= kparams->vtcm_size); - const size_t src0_row_size = src0->ne[0] * sizeof(float); - const size_t src0_row_stride = src0->nb[1]; - const size_t dst_row_size = dst->ne[0] * sizeof(float); - const size_t dst_row_stride = dst->nb[1]; + if (htp_tensor_is_extended(src1)) { + return HTP_STATUS_NO_SUPPORT; + } - // Aligned row sizes for VTCM - const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_stride, VLEN); - const size_t theta_cache_size_aligned = hex_round_up(src0->ne[0] * sizeof(float), 256); - - // Calculate spad sizes per thread - size_t src0_spad_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; - size_t dst_spad_per_thread = HTP_ROPE_SPAD_NROWS * dst_row_size_aligned; - size_t spad_per_thread = src0_spad_per_thread + dst_spad_per_thread; - - // Check if we fit in VTCM - size_t total_vtcm_needed = spad_per_thread * n_threads; - if (octx->ctx->vtcm_size < total_vtcm_needed) { - FARF(ERROR, "%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, total_vtcm_needed); - return HTP_STATUS_VTCM_TOO_SMALL; + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; } - octx->src0_spad.size_per_thread = src0_spad_per_thread; - octx->dst_spad.size_per_thread = dst_spad_per_thread; - octx->src0_spad.size = n_threads * src0_spad_per_thread; - octx->dst_spad.size = n_threads * dst_spad_per_thread; - octx->src1_spad.size = 0; + if (nrows == 0) { + return HTP_STATUS_OK; + } - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->src1_spad.data = NULL; octx->src1_spad.src = NULL; - octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL; + const uint32_t n_threads = octx->n_threads; + + const uint32_t ne0 = dst->ne[0]; + const size_t src0_row_size = src0->ne[0] * sizeof(float); + const size_t src0_row_stride = src0->nb[1]; + const size_t dst_row_size = dst->ne[0] * sizeof(float); + const size_t dst_row_stride = dst->nb[1]; struct htp_rope_context rctx; memset(&rctx, 0, sizeof(struct htp_rope_context)); - rctx.t_start = HAP_perf_get_qtimer_count(); - - rctx.octx = octx; + rctx.octx = octx; + rctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + rctx.spad_per_thread = kparams->spad_per_thread; + rctx.theta_cache_offset = kparams->theta_cache_offset; + + if (src2) { + dma_queue * dma_q = octx->ctx->dma[0]; + const size_t ff_size = src2->ne[0] * sizeof(float); + float * vtcm_freq_factors = (float *) (rctx.vtcm_base + kparams->freq_factors_offset); + dma_queue_push(dma_q, dma_make_data(vtcm_freq_factors, src2->data), + kparams->freq_factors_size, 0, ff_size, 1); + dma_queue_pop(dma_q); + rctx.freq_factors = vtcm_freq_factors; + } const int32_t * op_params = &octx->op_params[0]; rctx.n_dims = ((const int32_t *) op_params)[1]; @@ -693,31 +779,31 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { memcpy(&rctx.sections, (int32_t *) op_params + 11, sizeof(int) * 4); rctx.theta_scale = powf(rctx.freq_base, -2.0f / rctx.n_dims); + rctx.theta_powers[0] = 1.0f; + for (int j = 1; j < 32; j++) { + rctx.theta_powers[j] = rctx.theta_powers[j - 1] * rctx.theta_scale; + } + rctx.theta_scale_32 = rctx.theta_powers[31] * rctx.theta_scale; rope_corr_dims(rctx.n_dims, rctx.n_ctx_orig, rctx.freq_base, rctx.beta_fast, rctx.beta_slow, rctx.corr_dims); - rctx.src0_row_size = src0_row_size; - rctx.src0_row_stride = src0_row_stride; - rctx.dst_row_size = dst_row_size; - rctx.dst_row_stride = dst_row_stride; - rctx.src0_row_size_aligned = src0_row_size_aligned; - rctx.dst_row_size_aligned = dst_row_size_aligned; - rctx.theta_cache_offset = theta_cache_size_aligned; + rctx.src0_row_size = src0_row_size; + rctx.src0_row_stride = src0_row_stride; + rctx.dst_row_size = dst_row_size; + rctx.dst_row_stride = dst_row_stride; + rctx.src0_row_size_aligned = kparams->src0_row_size_aligned; - rctx.src0_nrows = src0_nrows; - rctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - - if (src0_nrows > 0) { - rctx.div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); - rctx.div_ne1 = init_fastdiv_values(dst->ne[1]); - } + rctx.src0_nrows = nrows; + rctx.nrows = nrows; + rctx.row_start = row_start; + rctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + rctx.div_ne2_ne1 = kparams->div_ne2_ne1; + rctx.div_ne1 = kparams->div_ne1; FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0, rctx.ext_factor, rctx.theta_scale, rctx.attn_factor); - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - worker_pool_run_func(octx->ctx->worker_pool, rope_job_f32, &rctx, n_threads); - } + work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, n_threads); return err; } diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.h new file mode 100644 index 000000000000..ee055ccbc19f --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/rope-ops.h @@ -0,0 +1,62 @@ +#ifndef HTP_ROPE_OPS_H +#define HTP_ROPE_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" + +#define HTP_ROPE_SPAD_BLOCK 8 +#define HTP_ROPE_SPAD_NSLOTS 4 +#define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS) + +struct htp_rope_kernel_params { + uint32_t n_threads; + uint32_t src0_nrows; + uint32_t src0_nrows_per_thread; + uint32_t vtcm_size; + uint32_t spad_per_thread; + uint32_t theta_cache_offset; + uint32_t src0_row_size_aligned; + uint32_t freq_factors_offset; + uint32_t freq_factors_size; + + struct fastdiv_values div_ne2_ne1; + struct fastdiv_values div_ne1; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_rope_vtcm_layout { + size_t total_bytes; + size_t bytes_per_thread; + size_t theta_cache_size_aligned; + size_t src0_row_size_aligned; + size_t freq_factors_size_aligned; +}; + +static inline void htp_rope_vtcm_layout_build( + struct htp_rope_vtcm_layout * layout, + uint32_t ne00, + uint32_t n_threads, + uint32_t n_freq_factors +) { + const size_t src0_row_size = ne00 * sizeof(float); + const size_t src0_row_size_aligned = hex_round_up((uint32_t) src0_row_size, 128); + const size_t theta_cache_size_aligned = hex_round_up((uint32_t) src0_row_size, 256); + const size_t freq_factors_size_aligned = hex_round_up(n_freq_factors * sizeof(float), 256); + + layout->src0_row_size_aligned = src0_row_size_aligned; + layout->theta_cache_size_aligned = theta_cache_size_aligned; + layout->freq_factors_size_aligned = freq_factors_size_aligned; + layout->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; + layout->total_bytes = layout->bytes_per_thread * n_threads + freq_factors_size_aligned; +} + +static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) { + return base + (slot * HTP_ROPE_SPAD_BLOCK) * row_size_aligned; +} + +#endif // HTP_ROPE_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.c index 340a497f7a2c..1d72538f176f 100644 --- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c @@ -18,6 +18,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -58,6 +59,9 @@ struct set_rows_context { const struct htp_set_rows_kernel_params * kparams; struct htp_set_rows_vtcm_layout vtcm_layout; uint8_t * vtcm_base; + uint32_t task_start; + uint32_t tasks; + uint32_t tasks_per_thread; }; #define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \ @@ -67,13 +71,13 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \ set_rows_preamble; \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - const uint32_t dr = kparams->tasks_per_thread; \ - const uint32_t ir0 = dr * ith; \ - if (ir0 >= kparams->total_tasks) { \ + const uint32_t dr = srctx->tasks_per_thread; \ + const uint32_t ir0 = srctx->task_start + dr * ith; \ + if (ir0 >= srctx->task_start + srctx->tasks) { \ return; \ } \ - const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ - dma_queue * dma_queue = octx->ctx->dma[ith]; \ + const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \ uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \ uint8_t * vtcm_dst = srctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \ @@ -86,14 +90,14 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig uint32_t pi03 = 0; \ for (uint32_t step = 0, spad_idx = 0; step < total_steps && spad_idx < 2; ++step, spad_idx++) { \ uint32_t i = ir0 + pi_step; \ - const uintptr_t src0_ptr = octx->src[0]->data + i*nb01 + pi02*nb02 + pi03*nb03; \ - dma_queue_push(dma_queue, \ - dma_make_ptr((void *)octx->dst->data, \ - vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \ + const dma_addr_t src0_data = octx->src[0]->data + i*nb01 + pi02*nb02 + pi03*nb03; \ + dma_queue_push(dma_q, \ + dma_make_data(octx->dst->data, \ + vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \ dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \ - dma_queue_push(dma_queue, \ - dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \ - (const void *)src0_ptr), \ + dma_queue_push(dma_q, \ + dma_make_data(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size, \ + src0_data), \ vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ pi_step++; \ if (pi_step == nrows_per_thread) { \ @@ -111,8 +115,8 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig uint32_t ci11_base = 0; \ uint32_t ci12_base = 0; \ for (uint32_t step = 0; step < total_steps; ++step) { \ - void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \ - void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \ + void * dst_spad = (void *) dma_queue_pop(dma_q).src; \ + void * src_spad = (void *) dma_queue_pop(dma_q).dst; \ uint32_t i = ir0 + ci_step; \ const uintptr_t src1_addr = octx->src[1]->data + i*nb10 + ci11_base*nb11 + ci12_base*nb12; \ const IDX_TYPE i1 = *(const IDX_TYPE *)src1_addr; \ @@ -124,21 +128,21 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig } \ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, step); \ if (valid_i1) { \ - const uintptr_t dst_ptr = octx->dst->data + target_i1*nb1 + ci02*nb2 + ci03*nb3; \ - dma_queue_push(dma_queue, \ - dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \ + const dma_addr_t dst_data = octx->dst->data + target_i1*nb1 + ci02*nb2 + ci03*nb3; \ + dma_queue_push(dma_q, \ + dma_make_data(dst_data, dst_spad), \ dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 1); \ } else { \ - dma_queue_push(dma_queue, \ - dma_make_ptr((void *)octx->dst->data, (const void *)dst_spad), \ + dma_queue_push(dma_q, \ + dma_make_data(octx->dst->data, dst_spad), \ dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \ } \ const uint32_t next_step = step + 2; \ if (next_step < total_steps) { \ uint32_t ni = ir0 + pi_step; \ - const uintptr_t psrc0_ptr = octx->src[0]->data + ni*nb01 + pi02*nb02 + pi03*nb03; \ - dma_queue_push(dma_queue, \ - dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \ + const dma_addr_t psrc0_data = octx->src[0]->data + ni*nb01 + pi02*nb02 + pi03*nb03; \ + dma_queue_push(dma_q, \ + dma_make_data(src_spad, psrc0_data), \ vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ pi_step++; \ if (pi_step == nrows_per_thread) { \ @@ -168,7 +172,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig } \ } \ } \ - dma_queue_flush(dma_queue); \ + dma_queue_flush(dma_q); \ } SET_ROWS_THREAD_DMA_FN(f32, int32_t, { hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); }) @@ -192,18 +196,44 @@ int op_set_rows(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (octx->src[1]->type != HTP_TYPE_I32 && octx->src[1]->type != HTP_TYPE_I64) { + if (htp_tensor_is_extended(octx->src[1])) { return HTP_STATUS_NO_SUPPORT; } + const struct htp_tensor * dst = octx->dst; + const uint32_t total_tasks = kparams->total_tasks; + + uint32_t task_start = 0; + uint32_t tasks = total_tasks; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && !htp_tensor_is_permuted(dst); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + task_start = range.start; + tasks = range.count; + } + + if (tasks == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + // l2fetch the src1 (indices) tensor in the main thread hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]); struct set_rows_context srctx; srctx.octx = octx; srctx.kparams = kparams; + srctx.task_start = task_start; + srctx.tasks = tasks; + srctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); - htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, kparams->n_threads); + htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_threads); srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; work_queue_func_t q_func = NULL; @@ -216,15 +246,15 @@ int op_set_rows(struct htp_ops_context * octx) { default: return HTP_STATUS_NO_SUPPORT; } - FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n_threads %d\n", + FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-threads %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - srctx.vtcm_layout.src0_bytes_per_thread * kparams->n_threads, - srctx.vtcm_layout.dst_bytes_per_thread * kparams->n_threads, - kparams->n_threads); + srctx.vtcm_layout.src0_bytes_per_thread * n_threads, + srctx.vtcm_layout.dst_bytes_per_thread * n_threads, + n_threads); - work_queue_run(octx->ctx->work_queue, q_func, &srctx, kparams->n_threads); + work_queue_run(octx->ctx->work_queue, q_func, &srctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.c b/ggml/src/ggml-hexagon/htp/softmax-ops.c index d78bcc0eb24e..48be5d725aab 100644 --- a/ggml/src/ggml-hexagon/htp/softmax-ops.c +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.c @@ -8,52 +8,49 @@ #include <math.h> #include <string.h> -#include "hex-dma.h" +#include "dma-queue.h" +#include "work-queue.h" #include "hvx-utils.h" #include "hex-fastdiv.h" +#include "hex-common.h" +#include "hex-profile.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" - -#define htp_softmax_preamble3 \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t ne10 = src1 ? src1->ne[0] : 1; \ - const uint32_t ne11 = src1 ? src1->ne[1] : 1; \ - const uint32_t ne12 = src1 ? src1->ne[2] : 1; \ - const uint32_t ne13 = src1 ? src1->ne[3] : 1; \ - \ - const uint32_t nb10 = src1 ? src1->nb[0] : 1; \ - const uint32_t nb11 = src1 ? src1->nb[1] : 1; \ - const uint32_t nb12 = src1 ? src1->nb[2] : 1; \ - const uint32_t nb13 = src1 ? src1->nb[3] : 1; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "htp/softmax-ops.h" +#include "hvx-flash-attn.h" struct htp_softmax_context { struct htp_ops_context * octx; + const struct htp_softmax_kernel_params * kparams; + + void * compute; + + dma_addr_t data_src0; + dma_addr_t data_src1; + dma_addr_t data_dst; + + uint8_t * vtcm_src0; + uint8_t * vtcm_src1; + uint8_t * vtcm_dst; + + uint32_t vtcm_src0_size_per_thread; + uint32_t vtcm_src1_size_per_thread; + uint32_t vtcm_dst_size_per_thread; + + uint32_t src0_spad_half_size; + uint32_t src1_spad_half_size; + uint32_t dst_spad_half_size; + + uint32_t src0_row_size_aligned; + uint32_t src1_row_size_aligned; + uint32_t dst_row_size_aligned; bool use_f16; - bool use_src1; uint32_t n_head; uint32_t n_head_log2; @@ -63,66 +60,78 @@ struct htp_softmax_context { float m0; float m1; - struct fastdiv_values fastdiv_ne01; - struct fastdiv_values fastdiv_ne02; - struct fastdiv_values fastdiv_ne12; // For mask broadcasting - struct fastdiv_values fastdiv_ne13; // For mask broadcasting + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne12; + struct fastdiv_values div_ne13; uint32_t src0_nrows_per_thread; + uint32_t row_start; + uint32_t nrows; + + float slopes[512] __attribute__((aligned(128))); }; -static void apply_mask(float * restrict wp0, - const float * restrict mp_f32, - const __fp16 * restrict mp_f16, - uint32_t ne00, - float slope, - bool use_f16) { - if (!mp_f32) { - return; - } - if (use_f16) { - for (uint32_t i = 0; i < ne00; ++i) { - wp0[i] += slope * (float) mp_f16[i]; - } - } else { - for (uint32_t i = 0; i < ne00; ++i) { - wp0[i] += slope * mp_f32[i]; - } - } -} +typedef void (*softmax_compute_fn_t)( + void * restrict dst, + const void * restrict src0, + const void * restrict mask, + uint32_t ne00, + float scale, + float slope +); -static void init_softmax_ctx(struct htp_softmax_context * smctx, struct htp_ops_context * octx) { - const struct htp_tensor * src0 = octx->src[0]; - const struct htp_tensor * src1 = octx->src[1]; +static void hvx_fast_softmax_prep_f16(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float scale, + const uint8_t * restrict mask, + float slope) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + const HVX_Vector * restrict v_mask = (const HVX_Vector *) mask; + + HVX_Vector scale_vec = hvx_vec_splat_f32(scale); + HVX_Vector slope_vec = hvx_vec_splat_f32(slope); - memset(smctx, 0, sizeof(struct htp_softmax_context)); + const int nvec_64 = num_elems / VLEN_FP16; + const int nloe_64 = num_elems % VLEN_FP16; - memcpy(&smctx->scale, (float *) octx->op_params, sizeof(float)); - memcpy(&smctx->max_bias, (float *) octx->op_params + 1, sizeof(float)); + #pragma unroll(2) + for (int i = 0; i < nvec_64; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_mask[i]); + HVX_Vector m0 = Q6_V_lo_W(p); + HVX_Vector m1 = Q6_V_hi_W(p); - smctx->n_head = src0->ne[2]; - smctx->n_head_log2 = 1u << (uint32_t) floor(log2(smctx->n_head)); + HVX_Vector s0 = v_src[2 * i]; + HVX_Vector s1 = v_src[2 * i + 1]; - smctx->m0 = powf(2.0f, -(smctx->max_bias) / smctx->n_head_log2); - smctx->m1 = powf(2.0f, -(smctx->max_bias / 2.0f) / smctx->n_head_log2); + HVX_Vector v0 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(s0, scale_vec), Q6_Vqf32_vmpy_VsfVsf(m0, slope_vec)); + HVX_Vector v1 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(s1, scale_vec), Q6_Vqf32_vmpy_VsfVsf(m1, slope_vec)); - smctx->use_src1 = (src1 != 0); - smctx->use_f16 = (src1 != 0) && (src1->type == HTP_TYPE_F16); + v_dst[2 * i] = Q6_Vsf_equals_Vqf32(v0); + v_dst[2 * i + 1] = Q6_Vsf_equals_Vqf32(v1); + } - smctx->octx = octx; + if (nloe_64 > 0) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_mask[nvec_64]); + HVX_Vector m0 = Q6_V_lo_W(p); - // Initialize fastdiv values - const uint32_t ne01 = src0->ne[1]; - const uint32_t ne02 = src0->ne[2]; + HVX_Vector s0 = v_src[2 * nvec_64]; + HVX_Vector v0 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(s0, scale_vec), Q6_Vqf32_vmpy_VsfVsf(m0, slope_vec)); - if (ne01 > 0) smctx->fastdiv_ne01 = init_fastdiv_values(ne01); - if (ne02 > 0) smctx->fastdiv_ne02 = init_fastdiv_values(ne02); + if (nloe_64 <= VLEN_FP32) { + hvx_vec_store_a(&v_dst[2 * nvec_64], nloe_64 * sizeof(float), Q6_Vsf_equals_Vqf32(v0)); + } else { + v_dst[2 * nvec_64] = Q6_Vsf_equals_Vqf32(v0); - const uint32_t ne12 = src1 ? src1->ne[2] : 1; - const uint32_t ne13 = src1 ? src1->ne[3] : 1; + HVX_Vector m1 = Q6_V_hi_W(p); + HVX_Vector s1 = v_src[2 * nvec_64 + 1]; + HVX_Vector v1 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(s1, scale_vec), Q6_Vqf32_vmpy_VsfVsf(m1, slope_vec)); - if (ne12 > 0) smctx->fastdiv_ne12 = init_fastdiv_values(ne12); - if (ne13 > 0) smctx->fastdiv_ne13 = init_fastdiv_values(ne13); + hvx_vec_store_a(&v_dst[2 * nvec_64 + 1], (nloe_64 - VLEN_FP32) * sizeof(float), Q6_Vsf_equals_Vqf32(v1)); + } + } } static void hvx_fast_softmax_prep_f32(const uint8_t * restrict src, @@ -131,57 +140,68 @@ static void hvx_fast_softmax_prep_f32(const uint8_t * restrict src, float scale, const uint8_t * restrict mask, float slope) { - const uint8_t * restrict src_curr = src; - uint8_t * restrict dst_curr = dst; - const uint8_t * restrict mask_curr = mask; + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + const HVX_Vector * restrict v_mask = (const HVX_Vector *) mask; HVX_Vector scale_vec = hvx_vec_splat_f32(scale); HVX_Vector slope_vec = hvx_vec_splat_f32(slope); - int step_of_1 = num_elems >> 5; + const int nvec = num_elems / VLEN_FP32; + const int nloe = num_elems % VLEN_FP32; #pragma unroll(4) - for (int i = 0; i < step_of_1; i++) { - HVX_Vector v1 = *(HVX_Vector *) src_curr; - - HVX_Vector v3 = *(HVX_Vector *) mask_curr; + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v3 = v_mask[i]; HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_vec); - HVX_Vector v4 = Q6_Vqf32_vmpy_VsfVsf(v3, slope_vec); - HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, v4); - *(HVX_Vector *) dst_curr = Q6_Vsf_equals_Vqf32(v5); + v_dst[i] = Q6_Vsf_equals_Vqf32(v5); + } + + if (nloe > 0) { + HVX_Vector v1 = v_src[nvec]; + HVX_Vector v3 = v_mask[nvec]; + + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_vec); + HVX_Vector v4 = Q6_Vqf32_vmpy_VsfVsf(v3, slope_vec); + HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, v4); - src_curr += VLEN; - dst_curr += VLEN; - mask_curr += VLEN; + hvx_vec_store_a(&v_dst[nvec], nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v5)); } } -static void hvx_fast_softmax_f32(const uint8_t * restrict src, uint8_t * restrict dst, uint8_t * restrict pad, const int num_elems) { - const HVX_Vector * restrict v_src = (HVX_Vector *) src; - HVX_Vector * restrict v_pad = (HVX_Vector *) pad; +static void hvx_fast_softmax_f32(const uint8_t * restrict src, uint8_t * restrict dst, const int num_elems) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; HVX_Vector * restrict v_dst = (HVX_Vector *) dst; - HVX_Vector sum_vec = Q6_V_vsplat_R(0x00000000); - HVX_Vector max_vec = hvx_vec_splat_f32(((const float *) src)[0]); - HVX_Vector zero_v = Q6_V_vzero(); - HVX_Vector one_v = hvx_vec_splat_f32(1.0); + const int nvec = num_elems / VLEN_FP32; + const int nloe = num_elems % VLEN_FP32; - int step_of_1 = num_elems >> 5; + HVX_Vector max_vec = hvx_vec_splat_f32(((const float *) src)[0]); - #pragma unroll(4) - for (int i = 0; i < step_of_1; i++) { + #pragma unroll(2) + for (int i = 0; i < nvec; i++) { HVX_Vector v1 = v_src[i]; max_vec = Q6_Vsf_vmax_VsfVsf(max_vec, v1); } - max_vec = hvx_vec_reduce_max_f32(max_vec); // replicated over all lanes + if (nloe > 0) { + HVX_VectorPred q_mask = Q6_Q_vsetq_R(nloe * sizeof(float)); + HVX_Vector neg_inf = hvx_vec_splat_f32(-INFINITY); + HVX_Vector v_tail = Q6_V_vmux_QVV(q_mask, v_src[nvec], neg_inf); + max_vec = Q6_Vsf_vmax_VsfVsf(max_vec, v_tail); + } - #pragma unroll(4) - for (int i = 0; i < step_of_1; i++) { + max_vec = hvx_vec_reduce_max_f32(max_vec); + + HVX_Vector sum_vec = Q6_V_vsplat_R(0x00000000); + + #pragma unroll(2) + for (int i = 0; i < nvec; i++) { HVX_Vector v1 = v_src[i]; HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, max_vec); @@ -189,205 +209,392 @@ static void hvx_fast_softmax_f32(const uint8_t * restrict src, uint8_t * restric sum_vec = Q6_Vqf32_vadd_VsfVsf(Q6_Vsf_equals_Vqf32(sum_vec), v3); - v_pad[i] = v3; + v_dst[i] = v3; + } + + if (nloe > 0) { + HVX_VectorPred q_mask = Q6_Q_vsetq_R(nloe * sizeof(float)); + HVX_Vector v1 = v_src[nvec]; + HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, max_vec); + HVX_Vector v3 = hvx_vec_exp_f32(Q6_Vsf_equals_Vqf32(v2)); + HVX_Vector v3_pad = Q6_V_vmux_QVV(q_mask, v3, Q6_V_vzero()); + + sum_vec = Q6_Vqf32_vadd_VsfVsf(Q6_Vsf_equals_Vqf32(sum_vec), v3_pad); + v_dst[nvec] = v3_pad; } - sum_vec = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_vec)); // replicated over all lanes + sum_vec = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_vec)); - HVX_VectorPred pos_sum = Q6_Q_vcmp_gt_VwVw(sum_vec, zero_v); + HVX_VectorPred pos_sum = Q6_Q_vcmp_gt_VwVw(sum_vec, Q6_V_vzero()); HVX_Vector v4 = hvx_vec_inverse_f32(sum_vec); - HVX_Vector scale_vec = Q6_V_vmux_QVV(pos_sum, v4, one_v); + HVX_Vector scale_vec = Q6_V_vmux_QVV(pos_sum, v4, hvx_vec_splat_f32(1.0f)); - #pragma unroll(4) - for (int i = 0; i < step_of_1; i++) { - HVX_Vector v1 = v_pad[i]; + #pragma unroll(2) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_dst[i]; HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_vec); v_dst[i] = Q6_Vsf_equals_Vqf32(v2); } + + if (nloe > 0) { + HVX_Vector v1 = v_dst[nvec]; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_vec); + hvx_vec_store_a(&v_dst[nvec], nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v2)); + } } -static float hvx_softmax_f32(const uint8_t * restrict src, uint8_t * restrict dst, uint8_t * restrict spad, const int num_elems, const float max) { - hvx_sub_scalar_f32(spad, src, max, num_elems); +static void compute_fast_softmax_f32_nomask( + void * restrict dst, + const void * restrict src0, + const void * restrict mask, + uint32_t ne00, + float scale, + float slope +) { + (void) mask; + (void) slope; + hvx_scale_f32((uint8_t *) dst, (const uint8_t *) src0, ne00, scale); + hvx_fast_softmax_f32((const uint8_t *) dst, (uint8_t *) dst, ne00); +} - hvx_exp_f32(dst, spad, num_elems, false); - return hvx_reduce_sum_f32(dst, num_elems); +static void compute_fast_softmax_f32_mask_f32( + void * restrict dst, + const void * restrict src0, + const void * restrict mask, + uint32_t ne00, + float scale, + float slope +) { + hvx_fast_softmax_prep_f32((const uint8_t *) src0, (uint8_t *) dst, ne00, scale, (const uint8_t *) mask, slope); + hvx_fast_softmax_f32((const uint8_t *) dst, (uint8_t *) dst, ne00); } -static void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) { - struct htp_softmax_context * smctx = (struct htp_softmax_context *) data; - struct htp_ops_context * octx = smctx->octx; +static void compute_fast_softmax_f32_mask_f16( + void * restrict dst, + const void * restrict src0, + const void * restrict mask, + uint32_t ne00, + float scale, + float slope +) { + hvx_fast_softmax_prep_f16((const uint8_t *) src0, (uint8_t *) dst, ne00, scale, (const uint8_t *) mask, slope); + hvx_fast_softmax_f32((const uint8_t *) dst, (uint8_t *) dst, ne00); +} + +static const softmax_compute_fn_t softmax_kernels[HTP_SOFTMAX_KERNEL_COUNT] = { + [HTP_SOFTMAX_KERNEL_NOMASK] = compute_fast_softmax_f32_nomask, + [HTP_SOFTMAX_KERNEL_MASK_F32] = compute_fast_softmax_f32_mask_f32, + [HTP_SOFTMAX_KERNEL_MASK_F16] = compute_fast_softmax_f32_mask_f16, +}; +static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data; + struct htp_ops_context * octx = smctx->octx; const struct htp_tensor * src0 = octx->src[0]; - const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - htp_softmax_preamble3; - - const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows + const uint32_t src0_nrows = smctx->nrows; const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); - // no work for this thread if (src0_start_row >= src0_end_row) { return; } - uint64_t qt = HAP_perf_get_qtimer_count(); + const dma_addr_t data_src0 = smctx->data_src0; + const dma_addr_t data_dst = smctx->data_dst; - int is_aligned = 1; - int opt_path = 0; + const size_t src0_row_size = src0->ne[0] * sizeof(float); + const size_t dst_row_size = src0->ne[0] * sizeof(float); - if (!hex_is_aligned((void *) src0->data, VLEN) || !hex_is_aligned((void *) dst->data, VLEN)) { - is_aligned = 0; - FARF(HIGH, "softmax-f32: unaligned addresses in elementwise op, possibly slower execution\n"); - } + uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread); + uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread); - // Only use the fast path when aligned AND row size is multiple of VLEN (128 bytes) - // The fast path (hvx_fast_softmax_f32) doesn't handle tail elements - // The non-opt path uses hvx_softmax_f32 which properly handles all sizes via its helper functions - if ((1 == is_aligned) && !(nb01 & (VLEN - 1))) { - opt_path = 1; - } + const size_t src0_vtcm_half = smctx->src0_spad_half_size; + const size_t dst_vtcm_half = smctx->dst_spad_half_size; - uint8_t * src0_spad_data = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * src1_spad_data = octx->src1_spad.data + (ith * octx->src1_spad.size_per_thread); - uint8_t * dst_spad_data = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); + dma_queue * dma_q = octx->ctx->dma[ith]; - float * wp0 = (float *) src0_spad_data; - float * wp1 = (float *) src1_spad_data; - float * wp2 = (float *) dst_spad_data; + for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) { + dma_addr_t cur_dst = data_dst + r * dst_row_size; + dma_addr_t cur_src0 = data_src0 + r * src0_row_size; + void * d_spad = dst_vtcm_base + idx * dst_vtcm_half; + void * s_spad = src0_vtcm_base + idx * src0_vtcm_half; - uint32_t prev_i2 = (uint32_t)-1; - float slope = 1.0f; + dma_queue_push(dma_q, dma_make_data(cur_dst, d_spad), + dst_row_size, smctx->dst_row_size_aligned, dst_row_size, 0); + dma_queue_push(dma_q, dma_make_data(s_spad, cur_src0), + smctx->src0_row_size_aligned, src0_row_size, src0_row_size, 1); + } + + softmax_compute_fn_t compute = (softmax_compute_fn_t) smctx->compute; + const uint32_t ne00 = src0->ne[0]; for (uint32_t r = src0_start_row; r < src0_end_row; ++r) { - uint32_t i1 = fastmodulo(r, ne01, &smctx->fastdiv_ne01); - uint32_t r_div_ne01 = fastdiv(r, &smctx->fastdiv_ne01); - uint32_t i2 = fastmodulo(r_div_ne01, ne02, &smctx->fastdiv_ne02); - uint32_t i3 = fastdiv(r_div_ne01, &smctx->fastdiv_ne02); - - // Map to original logic indices - // i01 = i1 - // i02 = i2 - // i03 = i3 - - const uint32_t i11 = i1; - // const uint32_t i12 = i2 % ne12; - // const uint32_t i13 = i3 % ne13; - - uint32_t i12, i13; - if (ne12 == ne02) { - i12 = i2; - } else { - i12 = fastmodulo(i2, ne12, &smctx->fastdiv_ne12); + void * d_spad = (void *) dma_queue_pop(dma_q).src; + void * s_spad = (void *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, r); + compute(d_spad, s_spad, NULL, ne00, smctx->scale, 1.0f); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, r); + + dma_addr_t cur_dst = data_dst + r * dst_row_size; + dma_queue_push(dma_q, dma_make_data(cur_dst, d_spad), + dst_row_size, smctx->dst_row_size_aligned, dst_row_size, 1); + + const uint32_t next_r = r + 2; + if (next_r < src0_end_row) { + dma_addr_t next_src0 = data_src0 + next_r * src0_row_size; + dma_queue_push(dma_q, dma_make_data(s_spad, next_src0), + smctx->src0_row_size_aligned, src0_row_size, src0_row_size, 1); } + } - if (ne13 == ne03) { - i13 = i3; - } else { - i13 = fastmodulo(i3, ne13, &smctx->fastdiv_ne13); - } + dma_queue_flush(dma_q); +} - // ALiBi - if (i2 != prev_i2) { - const uint32_t h = i2; // head - slope = (smctx->max_bias > 0.0f) ? h < smctx->n_head_log2 ? powf(smctx->m0, h + 1) : powf(smctx->m1, 2 * (h - smctx->n_head_log2) + 1) : 1.0f; - prev_i2 = i2; - } +static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data; + struct htp_ops_context * octx = smctx->octx; + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - float * sp = (float *) ((char *) src0->data + i1 * nb01 + i2 * nb02 + i3 * nb03); - float * dp = (float *) ((char *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3); + const uint32_t src0_nrows = smctx->nrows; + const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; - // broadcast the mask across rows - __fp16 * mp_f16 = (smctx->use_src1) ? (__fp16 *) ((char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13) : NULL; - float * mp_f32 = (smctx->use_src1) ? (float *) ((char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13) : NULL; + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); - if ((1 == opt_path) && (mp_f32) && !(smctx->use_f16)) { - hvx_fast_softmax_prep_f32((const uint8_t *) sp, (uint8_t *) wp0, ne00, smctx->scale, (const uint8_t *) mp_f32, slope); - hvx_fast_softmax_f32((const uint8_t *) wp0, (uint8_t *) dp, (uint8_t *) wp1, ne00); - } else if (1 == opt_path) { - hvx_scale_f32((uint8_t *) wp0, (const uint8_t *) sp, ne00, smctx->scale); - apply_mask(wp0, mp_f32, mp_f16, ne00, slope, smctx->use_f16); - hvx_fast_softmax_f32((const uint8_t *) wp0, (uint8_t *) dp, (uint8_t *) wp1, ne00); - } else { - // Non-optimized path: uses HVX helper functions that properly handle all tensor sizes - // including non-multiples of 32 (the HVX vector lane count for f32) - hvx_scale_f32((uint8_t *) wp0, (const uint8_t *) sp, ne00, smctx->scale); - apply_mask(wp0, mp_f32, mp_f16, ne00, slope, smctx->use_f16); - float max = hvx_reduce_max_f32((const uint8_t *) wp0, ne00); - float sum = hvx_softmax_f32((const uint8_t *) wp0, (uint8_t *) wp2, (uint8_t *) wp1, ne00, max); - sum = sum > 0.0 ? (1.0 / sum) : 1; - hvx_scale_f32((uint8_t *) dp, (const uint8_t *) wp2, ne00, sum); + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src0 = smctx->data_src0; + const dma_addr_t data_src1 = smctx->data_src1; + const dma_addr_t data_dst = smctx->data_dst; + + const size_t src0_row_size = src0->ne[0] * sizeof(float); + const size_t dst_row_size = src0->ne[0] * sizeof(float); + const size_t mask_row_size = smctx->use_f16 ? (src1->ne[0] * sizeof(__fp16)) : (src1->ne[0] * sizeof(float)); + + uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread); + uint8_t * src1_vtcm_base = smctx->vtcm_src1 + (ith * smctx->vtcm_src1_size_per_thread); + uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread); + + const size_t src0_vtcm_half = smctx->src0_spad_half_size; + const size_t src1_vtcm_half = smctx->src1_spad_half_size; + const size_t dst_vtcm_half = smctx->dst_spad_half_size; + + const uint32_t nb11 = src1->nb[1]; + const uint32_t nb12 = src1->nb[2]; + const uint32_t nb13 = src1->nb[3]; + + const uint32_t ne00 = src0->ne[0]; + const uint32_t ne01 = src0->ne[1]; + const uint32_t ne02 = src0->ne[2]; + const uint32_t ne03 = src0->ne[3]; + const uint32_t ne12 = src1->ne[2]; + const uint32_t ne13 = src1->ne[3]; + + const struct fastdiv_values * div_ne01 = &smctx->div_ne01; + const struct fastdiv_values * div_ne02 = &smctx->div_ne02; + const struct fastdiv_values * div_ne12 = &smctx->div_ne12; + const struct fastdiv_values * div_ne13 = &smctx->div_ne13; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) { + dma_addr_t cur_dst = data_dst + r * dst_row_size; + dma_addr_t cur_src0 = data_src0 + r * src0_row_size; + + uint32_t i1 = fastmodulo(r, ne01, div_ne01); + uint32_t r_div_ne01 = fastdiv(r, div_ne01); + uint32_t i2 = fastmodulo(r_div_ne01, ne02, div_ne02); + uint32_t i3 = fastdiv(r_div_ne01, div_ne02); + uint32_t i12 = (ne12 == ne02) ? i2 : fastmodulo(i2, ne12, div_ne12); + uint32_t i13 = (ne13 == ne03) ? i3 : fastmodulo(i3, ne13, div_ne13); + dma_addr_t cur_src1 = data_src1 + i1 * nb11 + i12 * nb12 + i13 * nb13; + + void * d_spad = dst_vtcm_base + idx * dst_vtcm_half; + void * s_spad = src0_vtcm_base + idx * src0_vtcm_half; + void * m_spad = src1_vtcm_base + idx * src1_vtcm_half; + + dma_queue_push(dma_q, dma_make_data(cur_dst, d_spad), + dst_row_size, smctx->dst_row_size_aligned, dst_row_size, 0); + dma_queue_push(dma_q, dma_make_data(s_spad, cur_src0), + smctx->src0_row_size_aligned, src0_row_size, src0_row_size, 1); + dma_queue_push(dma_q, dma_make_data(m_spad, cur_src1), + smctx->src1_row_size_aligned, mask_row_size, mask_row_size, 1); + } + + softmax_compute_fn_t compute = (softmax_compute_fn_t) smctx->compute; + const bool has_bias = smctx->max_bias > 0.0f; + uint32_t prev_i2 = (uint32_t)-1; + float slope = 1.0f; + + for (uint32_t r = src0_start_row; r < src0_end_row; ++r) { + void * d_spad = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * s_spad = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + void * m_spad = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + if (has_bias) { + uint32_t r_div_ne01 = fastdiv(r, div_ne01); + uint32_t i2 = fastmodulo(r_div_ne01, ne02, div_ne02); + if (i2 != prev_i2) { + slope = smctx->slopes[i2]; + prev_i2 = i2; + } + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, r); + compute(d_spad, s_spad, m_spad, ne00, smctx->scale, slope); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, r); + + dma_addr_t cur_dst = data_dst + r * dst_row_size; + dma_queue_push(dma_q, dma_make_data(cur_dst, d_spad), + dst_row_size, smctx->dst_row_size_aligned, dst_row_size, 1); + + const uint32_t next_r = r + 2; + if (next_r < src0_end_row) { + dma_addr_t next_src0 = data_src0 + next_r * src0_row_size; + + uint32_t ni1 = fastmodulo(next_r, ne01, div_ne01); + uint32_t nr_div_ne01 = fastdiv(next_r, div_ne01); + uint32_t ni2 = fastmodulo(nr_div_ne01, ne02, div_ne02); + uint32_t ni3 = fastdiv(nr_div_ne01, div_ne02); + uint32_t ni12 = (ne12 == ne02) ? ni2 : fastmodulo(ni2, ne12, div_ne12); + uint32_t ni13 = (ne13 == ne03) ? ni3 : fastmodulo(ni3, ne13, div_ne13); + dma_addr_t next_src1 = data_src1 + ni1 * nb11 + ni12 * nb12 + ni13 * nb13; + + dma_queue_push(dma_q, dma_make_data(s_spad, next_src0), + smctx->src0_row_size_aligned, src0_row_size, src0_row_size, 1); + dma_queue_push(dma_q, dma_make_data(m_spad, next_src1), + smctx->src1_row_size_aligned, mask_row_size, mask_row_size, 1); } } - qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt); - FARF(HIGH, "softmax-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u : opt %u f16 %u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, - ne0, ne1, ne2, ne3, opt_path, smctx->use_f16, (unsigned) qt); + dma_queue_flush(dma_q); } static int execute_op_softmax_f32(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; - const struct htp_tensor * src0 = octx->src[0]; - const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - struct htp_softmax_context smctx; const char * op_type = "softmax-f32"; - init_softmax_ctx(&smctx, octx); + const struct htp_softmax_kernel_params * kparams = + (const struct htp_softmax_kernel_params *) octx->kernel_params; - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - - smctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - - const size_t src0_row_size = src0->nb[1]; - const size_t src1_row_size = src0_row_size; - const size_t dst_row_size = dst->nb[1]; - - // VTCM scratchpads for all tensors - // 4 rows per thread, padded to HVX vector size - octx->src0_spad.size_per_thread = hex_round_up(4 * src0_row_size, 128); - octx->src1_spad.size_per_thread = hex_round_up(4 * src1_row_size, 128); - octx->dst_spad.size_per_thread = hex_round_up(4 * dst_row_size, 128); - - octx->src0_spad.size = octx->src0_spad.size_per_thread * n_threads; - octx->src1_spad.size = octx->src1_spad.size_per_thread * n_threads; - octx->dst_spad.size = octx->dst_spad.size_per_thread * n_threads; - - size_t spad_size = octx->src0_spad.size + octx->src1_spad.size + octx->dst_spad.size; - - if (src1) { - FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", - op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], octx->src0_spad.size, octx->src1_spad.size, - octx->dst_spad.size); - } else { - FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", op_type, - src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size); + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; } - // Make sure the reserved vtcm size is sufficient - if (octx->ctx->vtcm_size < spad_size) { - FARF(ERROR, "%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, spad_size); + if (kparams->kernel_id >= HTP_SOFTMAX_KERNEL_COUNT) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (octx->ctx->vtcm_size < (size_t) kparams->vtcm_size) { + FARF(ERROR, "%s : current VTCM reservation %zu is too small, needed %u\n", + op_type, octx->ctx->vtcm_size, kparams->vtcm_size); return HTP_STATUS_VTCM_TOO_SMALL; } - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->src1_spad.src = NULL; - octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; octx->dst_spad.src = NULL; + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t elem_size = sizeof(float); + const size_t dst_row_size = dst->nb[1]; + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + if (nrows < octx->n_threads) { + htp_ops_context_set_n_threads(octx, nrows ? nrows : 1); + } + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) return err; + const uint32_t n_threads = octx->n_threads; + uint8_t * const vtcm_base = (uint8_t *) octx->ctx->vtcm_base; - worker_pool_run_func(octx->ctx->worker_pool, softmax_job_f32, &smctx, n_threads); + const uint32_t off_src0 = 0; + const uint32_t off_dst = off_src0 + kparams->vtcm_src0_size_per_thread * kparams->n_threads; + const uint32_t off_src1 = off_dst + kparams->vtcm_dst_size_per_thread * kparams->n_threads; - return err; + struct htp_softmax_context smctx = { + .octx = octx, + .kparams = kparams, + .compute = (void *) softmax_kernels[kparams->kernel_id], + + .data_src0 = src0->data, + .data_src1 = kparams->use_src1 ? octx->src[1]->data : 0, + .data_dst = dst->data, + + .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, off_src0), + .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, off_dst), + .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, vtcm_base, off_src1, kparams->use_src1), + + .vtcm_src0_size_per_thread = kparams->vtcm_src0_size_per_thread, + .vtcm_src1_size_per_thread = kparams->vtcm_src1_size_per_thread, + .vtcm_dst_size_per_thread = kparams->vtcm_dst_size_per_thread, + + .src0_spad_half_size = kparams->src0_spad_half_size, + .src1_spad_half_size = kparams->src1_spad_half_size, + .dst_spad_half_size = kparams->dst_spad_half_size, + + .src0_row_size_aligned = kparams->src0_row_size_aligned, + .src1_row_size_aligned = kparams->src1_row_size_aligned, + .dst_row_size_aligned = kparams->dst_row_size_aligned, + + .use_f16 = kparams->use_f16 != 0, + + .n_head = kparams->n_head, + .n_head_log2 = kparams->n_head_log2, + + .scale = kparams->scale, + .max_bias = kparams->max_bias, + .m0 = kparams->m0, + .m1 = kparams->m1, + + .div_ne01 = kparams->div_ne01, + .div_ne02 = kparams->div_ne02, + .div_ne12 = kparams->div_ne12, + .div_ne13 = kparams->div_ne13, + + .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .row_start = row_start, + .nrows = nrows, + }; + + if (kparams->max_bias > 0.0f && kparams->use_src1) { + if (kparams->n_head > 512) { + return HTP_STATUS_INVAL_PARAMS; + } + for (uint32_t h = 0; h < kparams->n_head; h += 32) { + HVX_Vector v_slopes = hvx_alibi_slopes(h, 1, kparams->n_head_log2, kparams->m0, kparams->m1); + hvx_vmem(&smctx.slopes[h]) = v_slopes; + } + } + + work_queue_func_t task_func = kparams->use_src1 ? softmax_thread_mask_dma : softmax_thread_dma; + work_queue_run(octx->ctx->work_queue, task_func, &smctx, n_threads); + + return HTP_STATUS_OK; } int op_softmax(struct htp_ops_context * octx) { diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.h b/ggml/src/ggml-hexagon/htp/softmax-ops.h new file mode 100644 index 000000000000..8d976adb83ea --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.h @@ -0,0 +1,106 @@ +#ifndef HTP_SOFTMAX_OPS_H +#define HTP_SOFTMAX_OPS_H + +#include <stdint.h> +#include <stddef.h> +#include <stdbool.h> +#include <math.h> +#include "hex-fastdiv.h" +#include "hex-common.h" + +enum htp_softmax_kernel_id { + HTP_SOFTMAX_KERNEL_NOMASK = 0, + HTP_SOFTMAX_KERNEL_MASK_F32, + HTP_SOFTMAX_KERNEL_MASK_F16, + HTP_SOFTMAX_KERNEL_COUNT, +}; + +struct htp_softmax_kernel_params { + uint32_t n_threads; + uint32_t src0_nrows; + uint32_t src0_nrows_per_thread; + uint32_t vtcm_size; + + uint32_t vtcm_src0_size_per_thread; + uint32_t vtcm_src1_size_per_thread; + uint32_t vtcm_dst_size_per_thread; + + uint32_t src0_row_size_aligned; + uint32_t src1_row_size_aligned; + uint32_t dst_row_size_aligned; + + uint32_t src0_spad_half_size; + uint32_t src1_spad_half_size; + uint32_t dst_spad_half_size; + + uint32_t n_head; + uint32_t n_head_log2; + uint32_t use_src1; + uint32_t use_f16; + uint32_t kernel_id; + + float scale; + float max_bias; + float m0; + float m1; + + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne12; + struct fastdiv_values div_ne13; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_softmax_kernel_params) <= 128, "htp_softmax_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_softmax_kernel_params) <= 128, "htp_softmax_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_softmax_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + size_t off_src1; + + size_t src0_bytes_per_thread; + size_t dst_bytes_per_thread; + size_t src1_bytes_per_thread; + + size_t src0_spad_half_size; + size_t dst_spad_half_size; + size_t src1_spad_half_size; +}; + +static inline void htp_softmax_vtcm_layout_build( + struct htp_softmax_vtcm_layout * layout, + uint32_t ne00, + uint32_t ne10, + bool use_src1, + bool use_f16, + uint32_t n_threads +) { + size_t src0_row_size = ne00 * sizeof(float); + size_t dst_row_size = ne00 * sizeof(float); + size_t src1_row_size = use_src1 ? (ne10 * (use_f16 ? 2 : 4)) : 0; + + size_t src0_row_size_aligned = hex_round_up(src0_row_size, 128); + size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128); + size_t src1_row_size_aligned = use_src1 ? hex_round_up(src1_row_size, 128) : 0; + + layout->src0_spad_half_size = src0_row_size_aligned; + layout->dst_spad_half_size = dst_row_size_aligned; + layout->src1_spad_half_size = src1_row_size_aligned; + + // Double buffering: 2 half-buffers per thread + layout->src0_bytes_per_thread = src0_row_size_aligned * 2; + layout->dst_bytes_per_thread = dst_row_size_aligned * 2; + layout->src1_bytes_per_thread = src1_row_size_aligned * 2; + + layout->off_src0 = 0; + layout->off_dst = layout->off_src0 + layout->src0_bytes_per_thread * n_threads; + layout->off_src1 = layout->off_dst + layout->dst_bytes_per_thread * n_threads; + + layout->total_bytes = layout->off_src1 + layout->src1_bytes_per_thread * n_threads; +} + +#endif // HTP_SOFTMAX_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c index ae8e1a50495f..182982fcda66 100644 --- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c @@ -1,13 +1,16 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include <HAP_farf.h> -#include <HAP_perf.h> #include <string.h> +#include "hex-common.h" +#include "hex-profile.h" + #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-types.h" #include "hvx-utils.h" @@ -15,6 +18,7 @@ struct htp_solve_tri_context { struct htp_ops_context * octx; uint32_t jobs_per_thread; uint32_t total_jobs; + uint32_t job_start; uint32_t k_chunks; uint32_t col_block; }; @@ -89,11 +93,11 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t col_block = VLEN_FP32; const uint32_t k_full = (k / col_block) * col_block; - const uint32_t start_batch = sctx->jobs_per_thread * ith; - const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->total_jobs); + const uint32_t start_batch = sctx->job_start + sctx->jobs_per_thread * ith; + const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_batch); for (uint32_t batch = start_batch; batch < end_batch; ++batch) { const uint32_t i03 = batch / ne02; @@ -127,11 +131,10 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) end_batch); - FARF(HIGH, "solve-tri-batch %d/%d: A=(%ux%u) B=(%ux%u) batch %u:%u usec %u\n", - ith, nth, n, n, k, n, start_batch, end_batch, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + FARF(HIGH, "solve-tri-batch %d/%d: A=(%ux%u) B=(%ux%u) batch %u:%u\n", + ith, nth, n, n, k, n, start_batch, end_batch); } // Chunk-level thread: each job is one (batch, col_chunk) pair. @@ -148,11 +151,11 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t ne02 = src0->ne[2]; - const uint32_t start_job = sctx->jobs_per_thread * ith; - const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->total_jobs); + const uint32_t start_job = sctx->job_start + sctx->jobs_per_thread * ith; + const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_job); for (uint32_t job = start_job; job < end_job; ++job) { const uint32_t batch = job / sctx->k_chunks; @@ -161,16 +164,14 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t i03 = batch / ne02; const uint32_t i02 = batch - i03 * ne02; - const uint32_t col0 = chunk * sctx->col_block; - const uint32_t coln = MIN(sctx->col_block, k - col0); - const float * A_batch = (const float *) ((const uint8_t *) (uintptr_t) src0->data + i02 * src0->nb[2] + i03 * src0->nb[3]); const float * B_batch = (const float *) ((const uint8_t *) (uintptr_t) src1->data + i02 * src1->nb[2] + i03 * src1->nb[3]); float * X_batch = (float *) ((uint8_t *) (uintptr_t) dst->data + i02 * dst->nb[2] + i03 * dst->nb[3]); - const bool use_hvx = (coln >= 8); + const uint32_t col0 = chunk * sctx->col_block; + const uint32_t coln = MIN(sctx->col_block, k - col0); for (uint32_t row = 0; row < n; ++row) { const float diag = A_batch[row * n + row]; @@ -179,7 +180,7 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const float * A_row = A_batch + row * n; const float * B_row = B_batch + row * k; - if (use_hvx) { + if (coln >= 8) { solve_tri_row_hvx(A_row, B_row, X_batch, row, k, col0, coln, inv_diag); } else { solve_tri_row_scalar(A_row, B_row, X_batch, row, k, col0, coln, inv_diag); @@ -187,11 +188,10 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) end_job); - FARF(HIGH, "solve-tri-chunk %d/%d: A=(%ux%u) B=(%ux%u) job %u:%u usec %u\n", - ith, nth, n, n, k, n, start_job, end_job, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + FARF(HIGH, "solve-tri-chunk %d/%d: A=(%ux%u) B=(%ux%u) jobs %u:%u\n", + ith, nth, n, n, k, n, start_job, end_job); } int op_solve_tri(struct htp_ops_context * octx) { @@ -218,8 +218,8 @@ int op_solve_tri(struct htp_ops_context * octx) { return HTP_STATUS_INVAL_PARAMS; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; } const uint32_t k = src1->ne[0]; @@ -235,32 +235,64 @@ int op_solve_tri(struct htp_ops_context * octx) { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], batched); if (batched) { + uint32_t job_start = 0; + uint32_t njobs = total_batches; + + if (octx->ctx->mdev.count > 1) { + const uint32_t batch_size = dst->nb[2]; + const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, htp_tensor_mdev_data_aligned(dst) ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + job_start = range.start; + njobs = range.count; + } + + if (njobs == 0) { + return HTP_STATUS_OK; + } + // Batch-level parallelism - const uint32_t n_threads = MIN((uint32_t) octx->n_threads, total_batches); + const uint32_t n_threads = octx->n_threads; struct htp_solve_tri_context sctx = { .octx = octx, - .jobs_per_thread = (total_batches + n_threads - 1) / n_threads, - .total_jobs = total_batches, + .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .total_jobs = njobs, + .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - worker_pool_run_func(octx->ctx->worker_pool, solve_tri_batch_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->work_queue, solve_tri_batch_thread_f32, &sctx, n_threads); } else { // Chunk-level parallelism const uint32_t total_jobs = total_batches * k_chunks; - const uint32_t n_threads = MIN((uint32_t) octx->n_threads, MAX(total_jobs, 1)); + + uint32_t job_start = 0; + uint32_t njobs = total_jobs; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_jobs, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + job_start = range.start; + njobs = range.count; + } + + if (njobs == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; struct htp_solve_tri_context sctx = { .octx = octx, - .jobs_per_thread = (total_jobs + n_threads - 1) / n_threads, - .total_jobs = total_jobs, + .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .total_jobs = njobs, + .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - worker_pool_run_func(octx->ctx->worker_pool, solve_tri_chunk_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->work_queue, solve_tri_chunk_thread_f32, &sctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.c b/ggml/src/ggml-hexagon/htp/ssm-conv.c index a48bc9ed86b2..931aa406ea55 100644 --- a/ggml/src/ggml-hexagon/htp/ssm-conv.c +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.c @@ -4,7 +4,6 @@ #include <HAP_farf.h> #include <HAP_mem.h> -#include <HAP_perf.h> #include <HAP_ps.h> #include <hexagon_protos.h> #include <hexagon_types.h> @@ -15,121 +14,22 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" -#include "hex-dma.h" -#include "htp-ops.h" +#include "dma-queue.h" +#include "hex-profile.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-utils.h" - -#define htp_ssm_conv_tensors_preamble \ - const struct htp_tensor * restrict src0 = octx->src[0]; \ - const struct htp_tensor * restrict src1 = octx->src[1]; \ - const struct htp_tensor * restrict dst = octx->dst; \ - struct htp_spad * restrict src0_spad = &octx->src0_spad; \ - struct htp_spad * restrict src1_spad = &octx->src1_spad; \ - struct htp_spad * restrict dst_spad = &octx->dst_spad; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t ne10 = src1->ne[0]; \ - const uint32_t ne11 = src1->ne[1]; \ - const uint32_t ne12 = src1->ne[2]; \ - const uint32_t ne13 = src1->ne[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb10 = src1->nb[0]; \ - const uint32_t nb11 = src1->nb[1]; \ - const uint32_t nb12 = src1->nb[2]; \ - const uint32_t nb13 = src1->nb[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; +#include "ssm-conv.h" struct htp_ssm_conv_context { - struct htp_ops_context * octx; - uint32_t nrows_per_thread; - uint32_t d_inner_tile; - uint64_t t_start; + struct htp_ops_context * octx; + const struct htp_ssm_conv_kernel_params * kparams; + uint32_t nrows_per_thread; + uint32_t d_inner_tile; + uint32_t row_start; + uint32_t nrows; }; -#define htp_ssm_conv_preamble \ - struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; \ - struct htp_ops_context * octx = scctx->octx; \ - htp_ssm_conv_tensors_preamble; \ - dma_queue * dma_queue = octx->ctx->dma[ith]; - -// Scalar FP32 SSM_CONV implementation -static void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) { - htp_ssm_conv_preamble; - - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t d_conv = src1->ne[0]; - const uint32_t d_inner = src0->ne[1]; - const uint32_t n_t = dst->ne[1]; - const uint32_t n_s = dst->ne[2]; - - const uint32_t src0_stride_inner = src0->nb[1] / sizeof(float); // stride for inner dimension - const uint32_t src0_stride_seq = src0->nb[2] / sizeof(float); // stride for sequence dimension - const uint32_t src1_stride_inner = src1->nb[1] / sizeof(float); // stride for inner dimension - const uint32_t dst_stride_token = dst->nb[1] / sizeof(float); // stride for token dimension - const uint32_t dst_stride_seq = dst->nb[2] / sizeof(float); // stride for sequence dimension - - const float * src0_data = (const float *) src0->data; - const float * src1_data = (const float *) src1->data; - float * dst_data = (float *) dst->data; - - // Calculate row range for this thread - const uint32_t d_inner_per_thread = scctx->nrows_per_thread; - const uint32_t d_inner_start = d_inner_per_thread * ith; - const uint32_t d_inner_end = MIN(d_inner_start + d_inner_per_thread, d_inner); - - // No work for this thread - if (d_inner_start >= d_inner_end) { - return; - } - - for (uint32_t i3 = 0; i3 < n_s; ++i3) { - for (uint32_t i2 = 0; i2 < n_t; ++i2) { - for (uint32_t i1 = d_inner_start; i1 < d_inner_end; ++i1) { - float sumf = 0.0f; - - for (uint32_t i0 = 0; i0 < d_conv; ++i0) { - const uint32_t src0_idx = (i2 + i0) + i1 * src0_stride_inner + i3 * src0_stride_seq; - const uint32_t src1_idx = i0 + i1 * src1_stride_inner; - - sumf += src0_data[src0_idx] * src1_data[src1_idx]; - } - - const uint32_t dst_idx = i1 + i2 * dst_stride_token + i3 * dst_stride_seq; - dst_data[dst_idx] = sumf; - } - } - } - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "ssm-conv-f32 %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", - ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], d_inner_start, d_inner_end, - src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], - dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); -} - - // In-register 32x32 fp32 transpose using std 5-stage HVX vshuff butterfly. static inline void hvx_transpose_32x32_f32(HVX_Vector m[32]) { HVX_Vector tmp[32]; @@ -179,40 +79,69 @@ static inline void hvx_transpose_32x32_f32(HVX_Vector m[32]) { } } -// HVX FP32 SSM_CONV implementation - channel-vectorized HVX kernel with src0/src1 -// transposed into VTCM. -// -// VTCM layouts (per thread): -// src1_T : {d_inner_stride, d_conv} - staged once per launch (small). -// src0_T : {d_inner_tile, ncs} - staged per d_inner-tile. -// -// d_inner_tile is chosen so that per-thread VTCM stays under the budget. -// Each thread iterates ceil(d_inner_per_thread d_inner_tile) tiles serially. -#define HTP_SSM_CONV_VTCM_BUDGET (1u << 20) // 1 MiB per thread - -// Scalar transpose: src1 {d_conv, d_inner} (DDR) -> {d_inner_stride, d_conv} (VTCM) -static inline void transpose_src1(const float * src1_data, - uint32_t src1_stride_inner, - uint32_t i1_off, - uint32_t d_inner_per_thread, - uint32_t d_inner_stride, - uint32_t d_conv, - float * src1_T) { - for (uint32_t i = 0; i < d_inner_per_thread; ++i) { - const float * src_row = src1_data + (i1_off + i) * src1_stride_inner; +// HVX deinterleave for d_conv == 4: channel-major raw VTCM -> tap-major T VTCM +static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride) { + for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { + HVX_Vector v0 = *(const HVX_Vector *)(raw + (cb + 0) * 4); + HVX_Vector v1 = *(const HVX_Vector *)(raw + (cb + 8) * 4); + HVX_Vector v2 = *(const HVX_Vector *)(raw + (cb + 16) * 4); + HVX_Vector v3 = *(const HVX_Vector *)(raw + (cb + 24) * 4); + + HVX_VectorPair p01 = Q6_W_vdeal_VVR(v1, v0, -4); + HVX_VectorPair p23 = Q6_W_vdeal_VVR(v3, v2, -4); + + HVX_VectorPair p_w02 = Q6_W_vdeal_VVR(Q6_V_lo_W(p23), Q6_V_lo_W(p01), -4); + HVX_VectorPair p_w13 = Q6_W_vdeal_VVR(Q6_V_hi_W(p23), Q6_V_hi_W(p01), -4); + + *(HVX_Vector *)(T + 0 * d_inner_stride + cb) = Q6_V_lo_W(p_w02); + *(HVX_Vector *)(T + 1 * d_inner_stride + cb) = Q6_V_lo_W(p_w13); + *(HVX_Vector *)(T + 2 * d_inner_stride + cb) = Q6_V_hi_W(p_w02); + *(HVX_Vector *)(T + 3 * d_inner_stride + cb) = Q6_V_hi_W(p_w13); + } +} + +// HVX transpose for general d_conv <= 32: channel-major raw VTCM -> tap-major T VTCM +static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) { + uint32_t __attribute__((aligned(VLEN))) mask_buf[VLEN_FP32] = { 0 }; + for (uint32_t j = 0; j < d_conv; ++j) { + mask_buf[j] = 0xFFFFFFFF; + } + const HVX_Vector mask = *(const HVX_Vector *) mask_buf; + + for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { + const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_thread - cb); + HVX_Vector sub[32]; + for (uint32_t r = 0; r < cb_n; ++r) { + const float * ch_ptr = raw + (cb + r) * d_conv; + sub[r] = Q6_V_vand_VV(*(const HVX_UVector *) ch_ptr, mask); + } + for (uint32_t r = cb_n; r < 32; ++r) { + sub[r] = hvx_vec_splat_f32(0.0f); + } + + hvx_transpose_32x32_f32(sub); + for (uint32_t j = 0; j < d_conv; ++j) { - src1_T[j * d_inner_stride + i] = src_row[j]; + *(HVX_Vector *)(T + j * d_inner_stride + cb) = sub[j]; } } } -// HVX 32x32 src0 transpose: src0 {ncs, d_inner} (DDR) -> src0_T {d_inner_tile, ncs} (VTCM) +static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) { + if (d_conv == 4 && (d_inner_per_thread % VLEN_FP32 == 0)) { + hvx_ssm_conv_unpack_to_T_4(raw, T, d_inner_per_thread, d_inner_stride); + } else { + hvx_ssm_conv_unpack_to_T_gen(raw, T, d_inner_per_thread, d_inner_stride, d_conv); + } +} + +// HVX 32x32 src0 transpose for prefill: src0 {tile_n, ncs} (VTCM) -> src0_T {ncs, d_inner_tile} (VTCM) static inline void transpose_src0_block(const float * src0_block, uint32_t ncs, uint32_t cb_n, uint32_t d_inner_tile, float * src0_T_block_dst, - uint32_t cb /* dst column offset */) { + uint32_t cb) { const uint32_t T_TILE = VLEN_FP32; HVX_Vector __attribute__((aligned(VLEN))) sub[32]; @@ -220,20 +149,15 @@ static inline void transpose_src0_block(const float * src0_block, for (uint32_t t0 = 0; t0 < ncs; t0 += T_TILE) { const uint32_t t_n = MIN(T_TILE, ncs - t0); - // Load 32 rows (channels) of T_TILE samples; pad missing channels with zeros. + uint32_t __attribute__((aligned(VLEN))) mask_buf[VLEN_FP32] = { 0 }; + for (uint32_t k = 0; k < t_n; ++k) { + mask_buf[k] = 0xFFFFFFFF; + } + const HVX_Vector mask = *(const HVX_Vector *) mask_buf; + for (uint32_t r = 0; r < cb_n; ++r) { const float * src_row = src0_block + r * ncs + t0; - if (t_n == T_TILE) { - sub[r] = *(const HVX_UVector *) src_row; - } else { - HVX_Vector v = hvx_vec_splat_f32(0.0f); - hvx_vec_store_u(&v, t_n * sizeof(float), hvx_vec_splat_f32(0.0f)); - - float __attribute__((aligned(VLEN))) tmp[VLEN_FP32] = { 0 }; - for (uint32_t k = 0; k < t_n; ++k) tmp[k] = src_row[k]; - v = *(const HVX_Vector *) tmp; - sub[r] = v; - } + sub[r] = (t_n == T_TILE) ? *(const HVX_UVector *) src_row : Q6_V_vand_VV(*(const HVX_UVector *) src_row, mask); } for (uint32_t r = cb_n; r < T_TILE; ++r) { sub[r] = hvx_vec_splat_f32(0.0f); @@ -241,8 +165,6 @@ static inline void transpose_src0_block(const float * src0_block, hvx_transpose_32x32_f32(sub); - // Store transposed sub-tile to src0_T at offsets (t0 + j) * d_inner_tile + cb. - // Only write the valid t_n rows of the transposed result. for (uint32_t r = 0; r < t_n; ++r) { float * dst = src0_T_block_dst + (t0 + r) * d_inner_tile + cb; if (cb_n == T_TILE) { @@ -254,46 +176,165 @@ static inline void transpose_src0_block(const float * src0_block, } } -static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void *data) { - htp_ssm_conv_preamble; +// Single-row decode worker (n_t == 1) +static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void * data) { + struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; + struct htp_ops_context * octx = scctx->octx; + const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + const struct htp_tensor * restrict src0 = octx->src[0]; + const struct htp_tensor * restrict src1 = octx->src[1]; + const struct htp_tensor * restrict dst = octx->dst; - const uint32_t d_conv = src1->ne[0]; - const uint32_t d_inner = src0->ne[1]; - const uint32_t n_t = dst->ne[1]; - const uint32_t n_s = dst->ne[2]; - const uint32_t ncs = src0->ne[0]; + dma_queue * dma_q = octx->ctx->dma[ith]; - const uint32_t src0_stride_inner = src0->nb[1] / sizeof(float); - const uint32_t src0_stride_seq = src0->nb[2] / sizeof(float); - const uint32_t src1_stride_inner = src1->nb[1] / sizeof(float); - const uint32_t dst_stride_token = dst->nb[1] / sizeof(float); - const uint32_t dst_stride_seq = dst->nb[2] / sizeof(float); + const uint32_t d_conv = kparams->d_conv; + const uint32_t d_inner = kparams->d_inner; + const uint32_t n_s = kparams->n_s; const uint32_t dr = scctx->nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, d_inner); + const uint32_t ir0 = scctx->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); if (ir0 >= ir1) { return; } const uint32_t d_inner_per_thread = ir1 - ir0; - const uint32_t d_inner_stride = scctx->nrows_per_thread; + const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32); + + const size_t src0_stride_seq_bytes = src0->nb[2]; + const size_t dst_stride_seq_bytes = dst->nb[2]; + + uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; + uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; + + const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); + const size_t weight_raw_size = hex_round_up(weight_bytes, 128); + + float * src1_raw = (float *) src1_spad_base; + float * src1_T = (float *) (src1_spad_base + weight_raw_size); + + float * src0_raw = (float *) src0_spad_base; + float * src0_T = (float *) (src0_spad_base + weight_raw_size); + + float * dst_spad = (float *) dst_spad_base; + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + // 1. Fetch weights src1 from DDR into VTCM via DMA (DMA64-safe) + const dma_addr_t src1_ddr = src1->data + ir0 * d_conv * sizeof(float); + dma_queue_push(dma_q, dma_make_data((uint8_t *) src1_raw, src1_ddr), weight_bytes, weight_bytes, weight_bytes, 1); + dma_queue_pop(dma_q); + + // 2. Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); + hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); + + const size_t input_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); + const size_t output_bytes = (size_t) d_inner_per_thread * sizeof(float); + + // 3. Process each sequence + for (uint32_t s = 0; s < n_s; ++s) { + const dma_addr_t src0_ddr = src0->data + s * src0_stride_seq_bytes + ir0 * d_conv * sizeof(float); + dma_queue_push(dma_q, dma_make_data((uint8_t *) src0_raw, src0_ddr), input_bytes, input_bytes, input_bytes, 1); + dma_queue_pop(dma_q); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) s); + hvx_ssm_conv_unpack_to_T(src0_raw, src0_T, d_inner_per_thread, d_inner_stride, d_conv); + + for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { + const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_thread - cb); + HVX_Vector acc = hvx_vec_splat_f32(0.0f); + for (uint32_t j = 0; j < d_conv; ++j) { + HVX_Vector x = *(const HVX_Vector *)(src0_T + j * d_inner_stride + cb); + HVX_Vector w = *(const HVX_Vector *)(src1_T + j * d_inner_stride + cb); + acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w)); + } + HVX_Vector y = Q6_Vsf_equals_Vqf32(acc); + if (cb_n == VLEN_FP32) { + *(HVX_Vector *)(dst_spad + cb) = y; + } else { + hvx_vec_store_u(dst_spad + cb, cb_n * sizeof(float), y); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) s); + + const dma_addr_t dst_ddr = dst->data + s * dst_stride_seq_bytes + ir0 * sizeof(float); + dma_queue_push(dma_q, dma_make_data(dst_ddr, (uint8_t *) dst_spad), output_bytes, output_bytes, output_bytes, 1); + dma_queue_pop(dma_q); + } + + FARF(HIGH, "ssm-conv-f32-decode %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n", + ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, + src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], + dst->ne[2], dst->ne[3]); +} + +// Multi-token prefill worker (n_t > 1) +static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void * data) { + struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; + struct htp_ops_context * octx = scctx->octx; + const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams; + + const struct htp_tensor * restrict src0 = octx->src[0]; + const struct htp_tensor * restrict src1 = octx->src[1]; + const struct htp_tensor * restrict dst = octx->dst; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + const uint32_t d_conv = kparams->d_conv; + const uint32_t d_inner = kparams->d_inner; + const uint32_t n_t = kparams->n_t; + const uint32_t n_s = kparams->n_s; + const uint32_t ncs = src0->ne[0]; + + const uint32_t dr = scctx->nrows_per_thread; + const uint32_t ir0 = scctx->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); + + if (ir0 >= ir1) { + return; + } + + const uint32_t d_inner_per_thread = ir1 - ir0; + const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32); const uint32_t d_inner_tile = scctx->d_inner_tile; - const float * src0_data = (const float *) src0->data; - const float * src1_data = (const float *) src1->data; - float * dst_data = (float *) dst->data; + const size_t src0_stride_inner_bytes = src0->nb[1]; + const size_t src0_stride_seq_bytes = src0->nb[2]; + const size_t dst_stride_token_bytes = dst->nb[1]; + const size_t dst_stride_seq_bytes = dst->nb[2]; + + uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; + uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; - // Per-thread VTCM regions. - float * src0_T = (float *)(octx->src0_spad.data + ith * octx->src0_spad.size_per_thread); - float * src1_T = (float *)(octx->src1_spad.data + ith * octx->src1_spad.size_per_thread); + const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); + const size_t weight_raw_size = hex_round_up(weight_bytes, 128); - // Stage src1 weights once into VTCM in {d_inner_stride, d_conv} layout. - transpose_src1(src1_data, src1_stride_inner, ir0, d_inner_per_thread, d_inner_stride, d_conv, src1_T); + float * src1_raw = (float *) src1_spad_base; + float * src1_T = (float *) (src1_spad_base + weight_raw_size); + + const size_t src0_tile_raw_bytes = hex_round_up(d_inner_tile * ncs * sizeof(float), 128); + float * src0_tile_raw = (float *) src0_spad_base; + float * src0_T = (float *) (src0_spad_base + src0_tile_raw_bytes); + + float * dst_tile = (float *) dst_spad_base; + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + // 1. Fetch weights src1 from DDR into VTCM via DMA (DMA64-safe) + const dma_addr_t src1_ddr = src1->data + ir0 * d_conv * sizeof(float); + dma_queue_push(dma_q, dma_make_data((uint8_t *) src1_raw, src1_ddr), weight_bytes, weight_bytes, weight_bytes, 1); + dma_queue_pop(dma_q); + + // 2. Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); + hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); const uint32_t C_TILE = VLEN_FP32; @@ -301,14 +342,24 @@ static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void for (uint32_t tile_off = 0; tile_off < d_inner_per_thread; tile_off += d_inner_tile) { const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off); - // Place src0 chunk into VTCM in {d_inner_tile, ncs} layout. - const float * src0_block = src0_data + i3 * src0_stride_seq + (ir0 + tile_off) * src0_stride_inner; + // Fetch src0 chunk from DDR to VTCM via 2D DMA + const dma_addr_t src0_tile_ddr = src0->data + + i3 * src0_stride_seq_bytes + + (ir0 + tile_off) * src0_stride_inner_bytes; + const size_t row_bytes = ncs * sizeof(float); + + dma_queue_push(dma_q, dma_make_data((uint8_t *) src0_tile_raw, src0_tile_ddr), + row_bytes, src0_stride_inner_bytes, row_bytes, tile_n); + dma_queue_pop(dma_q); + // Transpose src0 chunk in VTCM into {d_inner_tile, ncs} layout + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off); for (uint32_t cb = 0; cb < tile_n; cb += C_TILE) { const uint32_t cb_n = MIN(C_TILE, tile_n - cb); - transpose_src0_block(src0_block + cb * src0_stride_inner, ncs, cb_n, d_inner_tile, src0_T, cb); + transpose_src0_block(src0_tile_raw + cb * ncs, ncs, cb_n, d_inner_tile, src0_T, cb); } + // Compute convolution for (uint32_t t = 0; t < n_t; ++t) { for (uint32_t cb = 0; cb < tile_n; cb += C_TILE) { const uint32_t cb_n = MIN(C_TILE, tile_n - cb); @@ -319,97 +370,115 @@ static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void HVX_Vector w = *(const HVX_Vector *) (src1_T + j * d_inner_stride + tile_off + cb); acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w)); } - HVX_Vector res = Q6_Vsf_equals_Vqf32(acc); - float * dst_ptr = dst_data + i3 * dst_stride_seq + t * dst_stride_token + (ir0 + tile_off + cb); + HVX_Vector y = Q6_Vsf_equals_Vqf32(acc); + float * dst_tile_ptr = dst_tile + t * tile_n + cb; if (cb_n == C_TILE) { - *(HVX_UVector *) dst_ptr = res; + *(HVX_Vector *) dst_tile_ptr = y; } else { - hvx_vec_store_u(dst_ptr, cb_n * sizeof(float), res); + hvx_vec_store_u(dst_tile_ptr, cb_n * sizeof(float), y); } } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off); + + // Writeback dst_tile from VTCM to DDR via 2D DMA + const dma_addr_t dst_tile_ddr = dst->data + + i3 * dst_stride_seq_bytes + + (ir0 + tile_off) * sizeof(float); + const size_t dst_row_bytes = tile_n * sizeof(float); + + dma_queue_push(dma_q, dma_make_data(dst_tile_ddr, (uint8_t *) dst_tile), + dst_stride_token_bytes, dst_row_bytes, dst_row_bytes, n_t); + dma_queue_pop(dma_q); } } - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "ssm-conv-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) tile=%u * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", - ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, d_inner_tile, + FARF(HIGH, "ssm-conv-f32-prefill %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n", + ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], - dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[2], dst->ne[3]); } int op_ssm_conv_f32(struct htp_ops_context * octx) { - htp_ssm_conv_tensors_preamble; + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) { - FARF(ERROR, "ssm_conv: only (F32 x F32 -> F32) OPs supported"); return HTP_STATUS_NO_SUPPORT; } - struct htp_ssm_conv_context scctx = { 0 }; - scctx.octx = octx; - - const uint32_t d_conv = src1->ne[0]; - const uint32_t d_inner = src0->ne[1]; - const uint32_t n_t = dst->ne[1]; // tokens per sequence - const uint32_t n_s = dst->ne[2]; // number of sequences in the batch - - const uint32_t n_threads = MIN(octx->n_threads, d_inner); - - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - uint32_t use_hvx = 0; - if (d_inner >= VLEN_FP32 && n_t >= VLEN_FP32) { - use_hvx = 1; - } - - scctx.nrows_per_thread = hex_round_up((d_inner + n_threads - 1) / n_threads, VLEN_FP32); - - const uint32_t d_inner_per_thread = scctx.nrows_per_thread; - const uint32_t ncs = src0->ne[0]; - - const uint32_t src1_T_size = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 256); - const uint32_t src0_T_max = HTP_SSM_CONV_VTCM_BUDGET > src1_T_size ? HTP_SSM_CONV_VTCM_BUDGET - src1_T_size : 0; - - uint32_t d_inner_tile = (src0_T_max / sizeof(float)) / ncs; - d_inner_tile -= (d_inner_tile % VLEN_FP32); - if (d_inner_tile == 0) { - FARF(HIGH, "ssm_conv-f32: inner tile rounds to 0 (ncs=%u), falling back to scalar\n", ncs); - use_hvx = 0; - } else { - scctx.d_inner_tile = d_inner_tile; + const struct htp_ssm_conv_kernel_params * kparams = (const struct htp_ssm_conv_kernel_params *) octx->kernel_params; - octx->src0_spad.size_per_thread = hex_round_up(d_inner_tile * ncs * sizeof(float), 256); - octx->src1_spad.size_per_thread = src1_T_size; - octx->dst_spad.size_per_thread = 0; - octx->src0_spad.size = octx->src0_spad.size_per_thread * n_threads; - octx->src1_spad.size = octx->src1_spad.size_per_thread * n_threads; - octx->dst_spad.size = 0; + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } - octx->src0_spad.data = octx->ctx->vtcm_base; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; - octx->src0_spad.src = NULL; - octx->src1_spad.src = NULL; + uint32_t row_start = 0; + uint32_t nrows = kparams->d_inner; + + if (octx->ctx->mdev.count > 1) { + const uint32_t elems_per_chunk = VLEN_FP32; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + kparams->d_inner, + htp_tensor_mdev_data_aligned(dst) ? elems_per_chunk : 0, + octx->ctx->mdev.idx, + octx->ctx->mdev.count, + &octx->ctx->mdev.count_div + ); + row_start = range.start; + nrows = range.count; + } - const size_t total_spad = octx->src0_spad.size + octx->src1_spad.size; - if (total_spad > octx->ctx->vtcm_size) { - FARF(HIGH, "ssm_conv-f32: scratchpad %zu exceeds VTCM %zu, falling back to scalar\n", - total_spad, octx->ctx->vtcm_size); - use_hvx = 0; - } - } + if (nrows == 0) { + return HTP_STATUS_OK; + } - FARF(HIGH, "ssm-conv-f32: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_hvx %d\n", src0->ne[0], - src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], - dst->ne[1], dst->ne[2], dst->ne[3], use_hvx); + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } - if (use_hvx) { - worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32_hvx, &scctx, n_threads); - } else { - worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32, &scctx, n_threads); - } + const uint32_t n_threads = octx->n_threads; + + octx->src0_spad.size_per_thread = kparams->vtcm_src0_size_per_thread; + octx->src1_spad.size_per_thread = kparams->vtcm_src1_size_per_thread; + octx->dst_spad.size_per_thread = kparams->vtcm_dst_size_per_thread; + + octx->src0_spad.size = kparams->vtcm_src0_size; + octx->src1_spad.size = kparams->vtcm_src1_size; + octx->dst_spad.size = kparams->vtcm_dst_size; + + octx->src0_spad.data = octx->ctx->vtcm_base; + octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; + octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; + octx->src0_spad.src = NULL; + octx->src1_spad.src = NULL; + octx->dst_spad.src = NULL; + + const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, VLEN_FP32); + + struct htp_ssm_conv_context scctx = { + .octx = octx, + .kparams = kparams, + .nrows_per_thread = d_inner_per_thread, + .d_inner_tile = kparams->d_inner_tile, + .row_start = row_start, + .nrows = nrows, + }; + + FARF(HIGH, "ssm-conv-f32: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : mode %s\n", + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], + src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + kparams->n_t == 1 ? "decode" : "prefill"); + + if (kparams->n_t == 1) { + work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_decode, &scctx, n_threads); + } else { + work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_prefill, &scctx, n_threads); } return HTP_STATUS_OK; @@ -418,16 +487,10 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { int op_ssm_conv(struct htp_ops_context * octx) { const struct htp_tensor * dst = octx->dst; - int err = HTP_STATUS_OK; - switch (dst->type) { case HTP_TYPE_F32: - err = op_ssm_conv_f32(octx); - break; + return op_ssm_conv_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.h b/ggml/src/ggml-hexagon/htp/ssm-conv.h new file mode 100644 index 000000000000..be62d7bf5127 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.h @@ -0,0 +1,40 @@ +#ifndef HTP_SSM_CONV_H +#define HTP_SSM_CONV_H + +#include <stdint.h> + +#include "hex-fastdiv.h" +#include "htp-ops.h" + +struct htp_ssm_conv_kernel_params { + uint32_t n_threads; + uint32_t d_conv; + uint32_t d_inner; + uint32_t n_t; + uint32_t n_s; + uint32_t d_inner_per_thread; + uint32_t d_inner_tile; + + uint32_t src0_row_size_aligned; + uint32_t src1_row_size_aligned; + uint32_t dst_row_size_aligned; + + uint32_t vtcm_src0_size_per_thread; + uint32_t vtcm_src1_size_per_thread; + uint32_t vtcm_dst_size_per_thread; + + uint32_t vtcm_src0_size; + uint32_t vtcm_src1_size; + uint32_t vtcm_dst_size; + uint32_t vtcm_size; + + struct fastdiv_values div_n_threads; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_ssm_conv_kernel_params) <= 128, "htp_ssm_conv_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_ssm_conv_kernel_params) <= 128, "htp_ssm_conv_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_SSM_CONV_H diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c index 874c41ab2ac7..9b8e04a0fa55 100644 --- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c @@ -8,40 +8,43 @@ #include <string.h> #include <math.h> -#include "hex-dma.h" +#include "dma-queue.h" #include "hvx-utils.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #define sum_rows_preamble \ const struct htp_tensor *src0 = octx->src[0]; \ const struct htp_tensor *dst = octx->dst; \ \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t nb00 = src0->nb[0]; \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb0 = dst->nb[0]; \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ struct sum_rows_context { + struct htp_ops_context * octx; const uint8_t * src_data; uint8_t * dst_data; uint32_t ne00; @@ -76,6 +79,9 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) // Calculate actual number of rows for this thread const uint32_t n_rows = end_row - start_row; + struct htp_thread_trace * tr = &smctx->octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); + for (uint32_t ir = 0; ir < n_rows; ir++) { const float * restrict src_local = src_th + (ir * (src_stride / sizeof(float))); @@ -89,6 +95,8 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) dst_th[ir] = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00); } } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); } int op_sum_rows(struct htp_ops_context * octx) { @@ -98,13 +106,30 @@ int op_sum_rows(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + const uint32_t src0_nrows = ne01 * ne02 * ne03; + const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { return HTP_STATUS_OK; } - const uint32_t src0_nrows = ne01 * ne02 * ne03; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - const uint32_t rows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + const uint32_t n_threads = octx->n_threads; + const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); bool opt_path = false; if ((0 == hex_is_aligned((void *) src0->data, VLEN)) && !(nb01 & (VLEN - 1))) { @@ -112,17 +137,18 @@ int op_sum_rows(struct htp_ops_context * octx) { } struct sum_rows_context smctx = { - .src_data = (const uint8_t *) src0->data, - .dst_data = (uint8_t *) dst->data, + .octx = octx, + .src_data = (const uint8_t *) src0->data + row_start * nb01, + .dst_data = (uint8_t *) dst->data + row_start * nb1, .ne00 = ne00, .src_stride = nb01, .dst_stride = nb1, .rows_per_thread = rows_per_thread, - .total_rows = src0_nrows, + .total_rows = nrows, .opt_path = opt_path, }; - worker_pool_run_func(octx->ctx->worker_pool, sum_rows_thread_f32, &smctx, n_threads); + work_queue_run(octx->ctx->work_queue, sum_rows_thread_f32, &smctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 5e62b4a9bd33..9a1479e29288 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -8,7 +8,7 @@ #include <math.h> #include <string.h> -#include "hex-dma.h" +#include "dma-queue.h" #include "hex-fastdiv.h" #include "hvx-exp.h" #include "hvx-sigmoid.h" @@ -23,13 +23,47 @@ #include "htp-vtcm.h" #include "hex-profile.h" +struct htp_unary_context; + +typedef void (*unary_compute_fn_t)(const void * restrict src, + void * restrict dst, + uint32_t num_rows, + const struct htp_unary_context * uctx); + +typedef void (*unary_rms_norm_mul_compute_fn_t)(const void * restrict src, + const void * restrict weight, + void * restrict dst, + uint32_t num_rows, + const struct htp_unary_context * uctx); + +typedef void (*unary_tri_compute_fn_t)(const void * restrict src, + void * restrict dst, + uint32_t num_rows, + uint32_t ir, + const struct htp_unary_context * uctx); + +typedef void (*unary_tile_compute_fn_t)(void * restrict dst, + const void * restrict src, + uint32_t tw, + const struct htp_unary_context * uctx); + +typedef void (*unary_tiled_tri_compute_fn_t)(const void * restrict src, + void * restrict dst, + uint32_t tile_elems, + uint32_t col_start, + uint32_t i01, + uint32_t ne0, + int32_t ttype); + struct htp_unary_context { struct htp_ops_context * octx; const struct htp_unary_kernel_params * kparams; - const uint8_t * data_src0; - const uint8_t * data_src1; // weight/scale tensor for RMS_NORM_MUL - uint8_t * data_dst; + void * compute; + + dma_addr_t data_src0; + dma_addr_t data_src1; // weight/scale tensor for RMS_NORM_MUL + dma_addr_t data_dst; size_t src0_data_row_size; // actual data bytes per row size_t src1_data_row_size; @@ -46,6 +80,7 @@ struct htp_unary_context { uint32_t block; uint32_t src0_nrows; uint32_t src0_nrows_per_thread; + uint32_t row_start; uint32_t nc; uint32_t col_tile; // tiled mode bool broadcast_weight; @@ -120,8 +155,8 @@ static inline uint32_t unary_block_size(uint32_t ir, const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; \ const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; -static void scale_f32(const float * restrict src, - float * restrict dst, +static void scale_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -138,8 +173,8 @@ static void scale_f32(const float * restrict src, } } -static void clamp_f32(const float * restrict src, - float * restrict dst, +static void clamp_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -156,8 +191,24 @@ static void clamp_f32(const float * restrict src, } } -static void rms_norm_f32(const float * restrict src, - float * restrict dst, +static void leaky_relu_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float negative_slope = 0.f; + memcpy(&negative_slope, &op_params[0], sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_leaky_relu_scalar_f32(dst_local, src_local, negative_slope, ne0); + } +} + +static void rms_norm_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -172,9 +223,9 @@ static void rms_norm_f32(const float * restrict src, } } -static void rms_norm_mul_f32(const float * restrict src, - const float * restrict weight, - float * restrict dst, +static void rms_norm_mul_f32(const void * restrict src, + const void * restrict weight, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -190,8 +241,8 @@ static void rms_norm_mul_f32(const float * restrict src, } } -static void norm_f32(const float * restrict src, - float * restrict dst, +static void norm_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -206,8 +257,8 @@ static void norm_f32(const float * restrict src, } } -static void sqr_f32(const float * restrict src, - float * restrict dst, +static void sqr_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -220,8 +271,8 @@ static void sqr_f32(const float * restrict src, } } -static void sqrt_f32(const float * restrict src, - float * restrict dst, +static void sqrt_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -234,8 +285,8 @@ static void sqrt_f32(const float * restrict src, } } -static void scale_f16(const _Float16 * restrict src, - _Float16 * restrict dst, +static void scale_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -252,8 +303,8 @@ static void scale_f16(const _Float16 * restrict src, } } -static void clamp_f16(const _Float16 * restrict src, - _Float16 * restrict dst, +static void clamp_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -270,8 +321,8 @@ static void clamp_f16(const _Float16 * restrict src, } } -static void rms_norm_f16(const _Float16 * restrict src, - _Float16 * restrict dst, +static void rms_norm_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -286,8 +337,8 @@ static void rms_norm_f16(const _Float16 * restrict src, } } -static void norm_f16(const _Float16 * restrict src, - _Float16 * restrict dst, +static void norm_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -302,8 +353,8 @@ static void norm_f16(const _Float16 * restrict src, } } -static void sqr_f16(const _Float16 * restrict src, - _Float16 * restrict dst, +static void sqr_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -316,8 +367,8 @@ static void sqr_f16(const _Float16 * restrict src, } } -static void sqrt_f16(const _Float16 * restrict src, - _Float16 * restrict dst, +static void sqrt_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -330,8 +381,8 @@ static void sqrt_f16(const _Float16 * restrict src, } } -static void abs_f16(const _Float16 * restrict src, - _Float16 * restrict dst, +static void abs_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -344,8 +395,8 @@ static void abs_f16(const _Float16 * restrict src, } } -static void log_f16(const _Float16 * restrict src, - _Float16 * restrict dst, +static void log_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -358,8 +409,8 @@ static void log_f16(const _Float16 * restrict src, } } -static void l2_norm_f16(const _Float16 * restrict src, - _Float16 * restrict dst, +static void l2_norm_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -374,8 +425,8 @@ static void l2_norm_f16(const _Float16 * restrict src, } } -static void neg_f32(const float * restrict src, - float * restrict dst, +static void neg_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -388,8 +439,8 @@ static void neg_f32(const float * restrict src, } } -static void exp_f32(const float * restrict src, - float * restrict dst, +static void exp_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -402,8 +453,8 @@ static void exp_f32(const float * restrict src, } } -static void sigmoid_f32(const float * restrict src, - float * restrict dst, +static void sigmoid_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -417,8 +468,8 @@ static void sigmoid_f32(const float * restrict src, } // silu(x) = x * sigmoid(x) -static void silu_f32(const float * restrict src, - float * restrict dst, +static void silu_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -433,8 +484,8 @@ static void silu_f32(const float * restrict src, } // gelu(x) = x * sigmoid(1.702 * x) (quick/sigmoid approximation, matches CPU GELU_QUICK reference) -static void gelu_f32(const float * restrict src, - float * restrict dst, +static void gelu_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -449,8 +500,8 @@ static void gelu_f32(const float * restrict src, } } -static void tri_f32(const float * restrict src, - float * restrict dst, +static void tri_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const uint32_t ir, const struct htp_unary_context * uctx) { @@ -480,7 +531,7 @@ static void tri_f32(const float * restrict src, } if (boundary > ne0) boundary = ne0; - // Full HVX vectors — each starts at a 128-byte aligned offset + // Full HVX vectors - each starts at a 128-byte aligned offset for (uint32_t i = 0; i < nvec; i++) { const uint32_t vec_start = i * VLEN_FP32; const uint32_t vec_end = vec_start + VLEN_FP32; @@ -534,8 +585,8 @@ static void tri_f32(const float * restrict src, } } -static void softplus_f32(const float * restrict src, - float * restrict dst, +static void softplus_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -547,14 +598,14 @@ static void softplus_f32(const float * restrict src, for (uint32_t i = 0; i < ne0; i++) { float x = src_f[i]; - // For x > 20: softplus(x) ≈ x (avoids exp overflow) + // For x > 20: softplus(x) ~ x (avoids exp overflow) dst_f[i] = (x > 20.0f) ? x : logf(1.0f + expf(x)); } } } -static void l2_norm_f32(const float * restrict src, - float * restrict dst, +static void l2_norm_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -562,15 +613,15 @@ static void l2_norm_f32(const float * restrict src, memcpy(&epsilon, op_params, sizeof(float)); for (uint32_t ir = 0; ir < num_rows; ir++) { - const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * src0_row_size_aligned)); - float * restrict dst_f = (float *)((uint8_t *)dst + (ir * dst_row_size_aligned)); + const uint8_t * restrict src_f = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_f = (uint8_t *)dst + (ir * dst_row_size_aligned); hvx_fast_l2_norm_f32((const uint8_t *)src_f, (uint8_t *)dst_f, ne0, epsilon); } } -static void tanh_f32(const float * restrict src, - float * restrict dst, +static void tanh_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -583,8 +634,8 @@ static void tanh_f32(const float * restrict src, } } -static void abs_f32(const float * restrict src, - float * restrict dst, +static void abs_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -597,8 +648,22 @@ static void abs_f32(const float * restrict src, } } -static void log_f32(const float * restrict src, - float * restrict dst, +static void relu_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_max_scalar_f32(dst_local, src_local, 0.0f, ne0); + } +} + +static void log_f32(const void * restrict src, + void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; @@ -611,359 +676,100 @@ static void log_f32(const float * restrict src, } } -#define DEFINE_UNARY_TASK_IMPL(NAME, TYPE, SUFFIX, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \ -static void unary_task_##SUFFIX##_##NAME(unsigned int nth, unsigned int ith, void * data) { \ - const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \ - struct htp_ops_context * octx = uctx->octx; \ - const struct htp_tensor * src = octx->src[0]; \ - const struct htp_tensor * dst = octx->dst; \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - \ - htp_unary_preamble; \ - \ - int32_t * op_params = octx->op_params; \ - uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; \ - \ - const size_t src0_data_row_size = uctx->src0_data_row_size; \ - const size_t dst_data_row_size = uctx->dst_data_row_size; \ - \ - const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; \ - const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; \ - \ - const uint32_t src0_nrows = uctx->src0_nrows; \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ - \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - const uint8_t * restrict data_src = uctx->data_src0; \ - const uint8_t * restrict data_src1 = uctx->data_src1; \ - uint8_t * restrict data_dst = uctx->data_dst; \ - \ - const struct htp_tensor * src1 = (IS_RMS_NORM_MUL) ? octx->src[1] : NULL; \ - const uint32_t nb11 = src1 ? src1->nb[1] : 0; \ - const uint32_t nb12 = src1 ? src1->nb[2] : 0; \ - const uint32_t nb13 = src1 ? src1->nb[3] : 0; \ - const uint32_t nb11_bc = (src1 && src1->ne[1] > 1) ? nb11 : 0; \ - const uint32_t nb12_bc = (src1 && src1->ne[2] > 1) ? nb12 : 0; \ - const uint32_t nb13_bc = (src1 && src1->ne[3] > 1) ? nb13 : 0; \ - const bool src1_contig = src1 ? ((nb12 == (size_t)ne01 * nb11) && (nb13 == (size_t)ne02 * nb12)) : false; \ - \ - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \ - uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread)) : NULL;\ - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); \ - \ - size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; \ - size_t src1_vtcm_half_size = uctx->src1_vtcm_half_size; \ - size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; \ - \ - const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && \ - (nb03 == (size_t)ne02 * nb02); \ - const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && \ - (nb3 == (size_t)ne2 * nb2); \ - \ - const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; \ - const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \ - const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \ - \ - const bool src1_needs_row_clip = (IS_RMS_NORM_MUL) && !uctx->broadcast_weight && !src1_contig; \ - const bool block_src0_contig = src0_contig && !src1_needs_row_clip; \ - const bool block_dst_contig = dst_contig && !src1_needs_row_clip; \ - \ - const uint32_t src0_max_block = block_src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); \ - const uint32_t dst_max_block = block_dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); \ - const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); \ - if (BLOCK == 0) { \ - FARF(ERROR, "unary-" #SUFFIX " : current VTCM reservation %zu is too small, needed at least %zu\n", \ - uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); \ - return; \ - } \ - \ - dma_queue * dma_queue = octx->ctx->dma[ith]; \ - \ - if ((IS_RMS_NORM_MUL) && uctx->broadcast_weight) { \ - dma_queue_push(dma_queue, dma_make_ptr(src1_vtcm_data, data_src1), \ - uctx->src1_row_size_aligned, 0, uctx->src1_data_row_size, 1); \ - dma_queue_flush(dma_queue); \ - } \ - \ - for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { \ - const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, \ - ne01, div_ne01); \ - \ - dma_queue_push(dma_queue, \ - dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), \ - nb1, dst_row_size_aligned, dst_data_row_size, 0); \ - \ - const size_t src0_off = src0_contig ? (ir * nb01) : \ - unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); \ - dma_queue_push(dma_queue, \ - dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), \ - src0_row_size_aligned, nb01, src0_data_row_size, block_size); \ - \ - if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \ - const size_t src1_off = src1_contig ? (ir * nb11) : \ - unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, nb13_bc); \ - dma_queue_push(dma_queue, \ - dma_make_ptr(src1_vtcm_data + (vtcm_idx * src1_vtcm_half_size), data_src1 + src1_off), \ - uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, block_size); \ - } \ - \ - ir += block_size; \ - } \ - \ - for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { \ - const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, \ - ne01, div_ne01); \ - \ - TYPE * dst_vtcm = (TYPE *) dma_queue_pop(dma_queue).src; \ - TYPE * src0_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst; \ - TYPE * src1_vtcm = NULL; \ - if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \ - src1_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst; \ - } \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ - CORE_EXPR; \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ - \ - const size_t dst_off = dst_contig ? (ir * nb1) : \ - unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3); \ - dma_queue_push(dma_queue, \ - dma_make_ptr(data_dst + dst_off, dst_vtcm), \ - nb1, dst_row_size_aligned, dst_data_row_size, block_size); \ - \ - const uint32_t next_ir = ir + block_size; \ - if (next_ir < src0_end_row) { \ - const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_src0_contig, \ - block_dst_contig, ne01, div_ne01); \ - const uint32_t pref_ir = next_ir + next_block_size; \ - if (pref_ir < src0_end_row) { \ - const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_src0_contig, \ - block_dst_contig, ne01, div_ne01); \ - const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) : \ - unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); \ - dma_queue_push(dma_queue, \ - dma_make_ptr(src0_vtcm, data_src + src0_pref_off), \ - src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); \ - \ - if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \ - const size_t src1_pref_off = src1_contig ? (pref_ir * nb11) : \ - unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, \ - nb13_bc); \ - dma_queue_push(dma_queue, \ - dma_make_ptr(src1_vtcm, data_src1 + src1_pref_off), \ - uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, pref_block_size); \ - } \ - } \ - } \ - ir += block_size; \ - } \ - \ - dma_queue_flush(dma_queue); \ +#// Pointwise unary ops on one column tile in VTCM. +static void tile_scale_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + float scale = 0.f; + float bias = 0.f; + memcpy(&scale, &uctx->octx->op_params[0], sizeof(float)); + memcpy(&bias, &uctx->octx->op_params[1], sizeof(float)); + hvx_scale_offset_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw, scale, bias); } -// F32 unary task: row-block DMA/VTCM plumbing, float-typed VTCM buffers. -#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \ - DEFINE_UNARY_TASK_IMPL(NAME, float, f32, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) - -DEFINE_UNARY_TASK(norm, false, false, norm_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(rms_norm, false, false, rms_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(rms_norm_mul, true, false, rms_norm_mul_f32(src0_vtcm, uctx->broadcast_weight ? (const float *) src1_vtcm_data : src1_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(scale, false, false, scale_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(clamp, false, false, clamp_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(sqr, false, false, sqr_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(sqrt, false, false, sqrt_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(unary_neg, false, false, neg_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(unary_exp, false, false, exp_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(unary_sigmoid, false, false, sigmoid_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(unary_silu, false, false, silu_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(unary_gelu, false, false, gelu_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(unary_softplus, false, false, softplus_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(unary_tanh, false, false, tanh_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(unary_abs, false, false, abs_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(unary_log, false, false, log_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK(tri, false, true, tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx)) - -// F16 unary tasks: same DMA/VTCM plumbing as DEFINE_UNARY_TASK, but VTCM buffers are -// _Float16-typed. None of the current F16 ops need RMS_NORM_MUL or TRI support. -DEFINE_UNARY_TASK_IMPL(norm, _Float16, f16, false, false, norm_f16(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK_IMPL(rms_norm, _Float16, f16, false, false, rms_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK_IMPL(scale, _Float16, f16, false, false, scale_f16(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK_IMPL(clamp, _Float16, f16, false, false, clamp_f16(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK_IMPL(sqr, _Float16, f16, false, false, sqr_f16(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK_IMPL(sqrt, _Float16, f16, false, false, sqrt_f16(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK_IMPL(l2_norm, _Float16, f16, false, false, l2_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK_IMPL(unary_abs, _Float16, f16, false, false, abs_f16(src0_vtcm, dst_vtcm, block_size, uctx)) -DEFINE_UNARY_TASK_IMPL(unary_log, _Float16, f16, false, false, log_f16(src0_vtcm, dst_vtcm, block_size, uctx)) - -// Apply a pointwise unary op to one column tile that is already in VTCM. -#define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \ -static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \ - const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \ - struct htp_ops_context * octx = uctx->octx; \ - const struct htp_tensor * src = octx->src[0]; \ - const struct htp_tensor * dst = octx->dst; \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - \ - htp_unary_preamble; \ - \ - int32_t * op_params = octx->op_params; \ - const uint32_t col_tile = uctx->col_tile; \ - \ - const uint32_t src0_nrows = uctx->src0_nrows; \ - const uint32_t src0_start_row = uctx->src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + uctx->src0_nrows_per_thread, src0_nrows); \ - \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - const uint8_t * restrict data_src = uctx->data_src0; \ - uint8_t * restrict data_dst = uctx->data_dst; \ - \ - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \ - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); \ - \ - const size_t src0_half = uctx->src0_vtcm_half_size; \ - const size_t dst_half = uctx->dst_vtcm_half_size; \ - \ - dma_queue * dmaq = octx->ctx->dma[ith]; \ - \ - const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; \ - const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \ - const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \ - const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; \ - \ - const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; \ - const int32_t tri_ttype = (IS_TRI) ? op_params[0] : 0; \ - \ - const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && \ - (nb03 == (size_t)ne02 * nb02); \ - const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && \ - (nb3 == (size_t)ne2 * nb2); \ - \ - const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; \ - \ - for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { \ - const uint32_t row = src0_start_row + t / tiles_per_row; \ - const uint32_t col = (t % tiles_per_row) * col_tile; \ - const uint32_t tw = MIN(col_tile, ne0 - col); \ - const size_t tb = (size_t) tw * sizeof(float); \ - const size_t soff = (src0_contig ? (row * nb01) : \ - unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) +\ - (size_t) col * sizeof(float); \ - \ - dma_queue_push(dmaq, dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); \ - dma_queue_push(dmaq, dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1);\ - } \ - \ - uint32_t row = src0_start_row; \ - uint32_t col = 0; \ - uint32_t tile_in_row = 0; \ - uint32_t i01 = fastmodulo(row, ne01, div_ne01); \ - \ - uint32_t prow = src0_start_row + fastdiv(2, div_tpr); \ - uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; \ - uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); \ - \ - for (uint32_t t = 0; t < total_tiles; t++) { \ - uint8_t * dst_vtcm = (uint8_t *) dma_queue_pop(dmaq).src; \ - uint8_t * src_vtcm = (uint8_t *) dma_queue_pop(dmaq).dst; \ - \ - const uint32_t tw = MIN(col_tile, ne0 - col); \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); \ - CORE_TILE_EXPR; \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); \ - \ - const size_t doff = (dst_contig ? (row * nb1) : \ - unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + \ - (size_t) col * sizeof(float); \ - const size_t tb = (size_t) tw * sizeof(float); \ - dma_queue_push(dmaq, dma_make_ptr(data_dst + doff, dst_vtcm), tb, tb, tb, 1); \ - \ - const uint32_t pt = t + 2; \ - if (pt < total_tiles) { \ - const uint32_t ptw = MIN(col_tile, ne0 - pcol); \ - const size_t ptb = (size_t) ptw * sizeof(float); \ - const size_t psoff = (src0_contig ? (prow * nb01) : \ - unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, \ - nb03)) + \ - (size_t) pcol * sizeof(float); \ - dma_queue_push(dmaq, dma_make_ptr(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); \ - } \ - \ - tile_in_row++; \ - col += col_tile; \ - if (tile_in_row == tiles_per_row) { \ - tile_in_row = 0; \ - col = 0; \ - row++; \ - i01++; \ - if (i01 == ne01) { \ - i01 = 0; \ - } \ - } \ - \ - ptile_in_row++; \ - pcol += col_tile; \ - if (ptile_in_row == tiles_per_row) { \ - ptile_in_row = 0; \ - pcol = 0; \ - prow++; \ - } \ - } \ - \ - dma_queue_flush(dmaq); \ +static void tile_clamp_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + float min = 0.f; + float max = 0.f; + memcpy(&min, &uctx->octx->op_params[0], sizeof(float)); + memcpy(&max, &uctx->octx->op_params[1], sizeof(float)); + hvx_clamp_scalar_f32((uint8_t *) dst, (const uint8_t *) src, min, max, tw); } -static inline void tile_scale_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw, const int32_t * op_params) { - float scale = 0.f; - float bias = 0.f; - memcpy(&scale, &op_params[0], sizeof(float)); - memcpy(&bias, &op_params[1], sizeof(float)); - hvx_scale_offset_f32_aa(dst_vtcm, src_vtcm, tw, scale, bias); +static void tile_leaky_relu_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + float negative_slope = 0.f; + memcpy(&negative_slope, &uctx->octx->op_params[0], sizeof(float)); + hvx_leaky_relu_scalar_f32((uint8_t *) dst, (const uint8_t *) src, negative_slope, tw); } -static inline void tile_clamp_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw, const int32_t * op_params) { - float min = 0.f; - float max = 0.f; - memcpy(&min, &op_params[0], sizeof(float)); - memcpy(&max, &op_params[1], sizeof(float)); - hvx_clamp_scalar_f32(dst_vtcm, src_vtcm, min, max, tw); +static void tile_sqr_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_sqr_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_sqrt_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_sqrt_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); } -static inline void tile_unary_softplus_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) { - const float * restrict sf = (const float *) src_vtcm; - float * restrict df = (float *) dst_vtcm; +static void tile_neg_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_scale_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw, -1.0f); +} + +static void tile_exp_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_exp_f32((uint8_t *) dst, (const uint8_t *) src, tw, false); +} + +static void tile_sigmoid_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_sigmoid_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_silu_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_sigmoid_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); + hvx_mul_f32_aaa((uint8_t *) dst, (const uint8_t *) src, (uint8_t *) dst, tw); +} + +static void tile_gelu_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_mul_scalar_f32((uint8_t *) dst, (const uint8_t *) src, 1.702f, tw); + hvx_sigmoid_f32_aa((uint8_t *) dst, (uint8_t *) dst, tw); + hvx_mul_f32_aaa((uint8_t *) dst, (const uint8_t *) src, (uint8_t *) dst, tw); +} + +static void tile_softplus_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + const float * restrict sf = (const float *) src; + float * restrict df = (float *) dst; for (uint32_t i = 0; i < tw; i++) { float x = sf[i]; df[i] = (x > 20.0f) ? x : logf(1.0f + expf(x)); } } -// silu(x) = x * sigmoid(x) -static inline void tile_silu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) { - hvx_sigmoid_f32_aa(dst_vtcm, src_vtcm, tw); - hvx_mul_f32_aaa(dst_vtcm, src_vtcm, dst_vtcm, tw); +static void tile_tanh_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_tanh_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); } -// gelu(x) = x * sigmoid(1.702 * x) (quick/sigmoid approximation, matches CPU GELU_QUICK reference) -static inline void tile_gelu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) { - hvx_mul_scalar_f32(dst_vtcm, src_vtcm, 1.702f, tw); - hvx_sigmoid_f32_aa(dst_vtcm, dst_vtcm, tw); - hvx_mul_f32_aaa(dst_vtcm, src_vtcm, dst_vtcm, tw); +static void tile_abs_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_abs_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_log_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_log_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_relu_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_max_scalar_f32((uint8_t *) dst, (const uint8_t *) src, 0.0f, tw); } -// Triangular mask applied to one column tile. Boundary is an absolute column index, so -// each vector compares against its absolute column position (col_start + i*VLEN_FP32). -static inline void tri_apply_tile_f32(const uint8_t * restrict src, uint8_t * restrict dst, - uint32_t tile_elems, uint32_t col_start, uint32_t i01, - uint32_t ne0, int32_t ttype) { +static void tri_apply_tile_f32(const void * restrict src, void * restrict dst, + uint32_t tile_elems, uint32_t col_start, uint32_t i01, + uint32_t ne0, int32_t ttype) { const HVX_Vector * restrict v_src = (const HVX_Vector *) src; HVX_Vector * restrict v_dst = (HVX_Vector *) dst; const HVX_Vector zero = hvx_vec_splat_f32(0.0f); @@ -1033,20 +839,623 @@ static inline void tri_apply_tile_f32(const uint8_t * restrict src, uint8_t * re } } -DEFINE_UNARY_TILED_TASK(scale, false, tile_scale_f32(dst_vtcm, src_vtcm, tw, op_params)) -DEFINE_UNARY_TILED_TASK(clamp, false, tile_clamp_f32(dst_vtcm, src_vtcm, tw, op_params)) -DEFINE_UNARY_TILED_TASK(sqr, false, hvx_sqr_f32_aa(dst_vtcm, src_vtcm, tw)) -DEFINE_UNARY_TILED_TASK(sqrt, false, hvx_sqrt_f32_aa(dst_vtcm, src_vtcm, tw)) -DEFINE_UNARY_TILED_TASK(unary_neg, false, hvx_scale_f32_aa(dst_vtcm, src_vtcm, tw, -1.0f)) -DEFINE_UNARY_TILED_TASK(unary_exp, false, hvx_exp_f32(dst_vtcm, src_vtcm, tw, false)) -DEFINE_UNARY_TILED_TASK(unary_sigmoid, false, hvx_sigmoid_f32_aa(dst_vtcm, src_vtcm, tw)) -DEFINE_UNARY_TILED_TASK(unary_silu, false, tile_silu_f32(dst_vtcm, src_vtcm, tw)) -DEFINE_UNARY_TILED_TASK(unary_gelu, false, tile_gelu_f32(dst_vtcm, src_vtcm, tw)) -DEFINE_UNARY_TILED_TASK(unary_softplus, false, tile_unary_softplus_f32(dst_vtcm, src_vtcm, tw)) -DEFINE_UNARY_TILED_TASK(unary_tanh, false, hvx_tanh_f32_aa(dst_vtcm, src_vtcm, tw)) -DEFINE_UNARY_TILED_TASK(unary_abs, false, hvx_abs_f32_aa(dst_vtcm, src_vtcm, tw)) -DEFINE_UNARY_TILED_TASK(unary_log, false, hvx_log_f32_aa(dst_vtcm, src_vtcm, tw)) -DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype)) +// 1. Standard row-block unary task (F32 and F16). +static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const size_t src0_data_row_size = uctx->src0_data_row_size; + const size_t dst_data_row_size = uctx->dst_data_row_size; + const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; + const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_dst = uctx->data_dst; + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + + const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; + const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; + + const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && + (nb03 == (size_t)ne02 * nb02); + const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && + (nb3 == (size_t)ne2 * nb2); + + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; + const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; + + const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); + const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); + const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); + if (BLOCK == 0) { + FARF(ERROR, "unary-row-block : current VTCM reservation %zu is too small, needed at least %zu\n", + uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); + return; + } + + dma_queue * dma_q = octx->ctx->dma[ith]; + + for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, + ne01, div_ne01); + + dma_queue_push(dma_q, + dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), + nb1, dst_row_size_aligned, dst_data_row_size, 0); + + const size_t src0_off = src0_contig ? (ir * nb01) : + unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), + src0_row_size_aligned, nb01, src0_data_row_size, block_size); + + ir += block_size; + } + + unary_compute_fn_t compute = (unary_compute_fn_t) uctx->compute; + + for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, + ne01, div_ne01); + + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); + compute(src0_vtcm, dst_vtcm, block_size, uctx); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); + + const size_t dst_off = dst_contig ? (ir * nb1) : + unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3); + dma_queue_push(dma_q, + dma_make_data(data_dst + dst_off, dst_vtcm), + nb1, dst_row_size_aligned, dst_data_row_size, block_size); + + const uint32_t next_ir = ir + block_size; + if (next_ir < src0_end_row) { + const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig, + dst_contig, ne01, div_ne01); + const uint32_t pref_ir = next_ir + next_block_size; + if (pref_ir < src0_end_row) { + const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig, + dst_contig, ne01, div_ne01); + const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) : + unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm, data_src + src0_pref_off), + src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); + } + } + ir += block_size; + } + + dma_queue_flush(dma_q); +} + +// 2. RMS_NORM_MUL row-block task with weight buffer. +static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const size_t src0_data_row_size = uctx->src0_data_row_size; + const size_t dst_data_row_size = uctx->dst_data_row_size; + const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; + const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_src1 = uctx->data_src1; + const dma_addr_t data_dst = uctx->data_dst; + + const struct htp_tensor * src1 = octx->src[1]; + const uint32_t nb11 = src1->nb[1]; + const uint32_t nb12 = src1->nb[2]; + const uint32_t nb13 = src1->nb[3]; + const uint32_t nb11_bc = (src1->ne[1] > 1) ? nb11 : 0; + const uint32_t nb12_bc = (src1->ne[2] > 1) ? nb12 : 0; + const uint32_t nb13_bc = (src1->ne[3] > 1) ? nb13 : 0; + const bool src1_contig = ((nb12 == (size_t)ne01 * nb11) && (nb13 == (size_t)ne02 * nb12)); + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); + uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread)) : NULL; + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + + const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; + const size_t src1_vtcm_half_size = uctx->src1_vtcm_half_size; + const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; + + const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && + (nb03 == (size_t)ne02 * nb02); + const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && + (nb3 == (size_t)ne2 * nb2); + + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; + const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; + + const bool src1_needs_row_clip = !uctx->broadcast_weight && !src1_contig; + const bool block_src0_contig = src0_contig && !src1_needs_row_clip; + const bool block_dst_contig = dst_contig && !src1_needs_row_clip; + + const uint32_t src0_max_block = block_src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); + const uint32_t dst_max_block = block_dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); + const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); + if (BLOCK == 0) { + FARF(ERROR, "unary-rms-norm-mul : current VTCM reservation %zu is too small, needed at least %zu\n", + uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); + return; + } + + dma_queue * dma_q = octx->ctx->dma[ith]; + + if (uctx->broadcast_weight) { + dma_queue_push(dma_q, dma_make_data(src1_vtcm_data, data_src1), + uctx->src1_row_size_aligned, 0, uctx->src1_data_row_size, 1); + dma_queue_flush(dma_q); + } + + for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, + ne01, div_ne01); + + dma_queue_push(dma_q, + dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), + nb1, dst_row_size_aligned, dst_data_row_size, 0); + + const size_t src0_off = src0_contig ? (ir * nb01) : + unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), + src0_row_size_aligned, nb01, src0_data_row_size, block_size); + + if (!uctx->broadcast_weight) { + const size_t src1_off = src1_contig ? (ir * nb11) : + unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, nb13_bc); + dma_queue_push(dma_q, + dma_make_data(src1_vtcm_data + (vtcm_idx * src1_vtcm_half_size), data_src1 + src1_off), + uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, block_size); + } + + ir += block_size; + } + + unary_rms_norm_mul_compute_fn_t compute = (unary_rms_norm_mul_compute_fn_t) uctx->compute; + + for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, + ne01, div_ne01); + + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + void * src1_vtcm = NULL; + if (!uctx->broadcast_weight) { + src1_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); + const void * w = uctx->broadcast_weight ? (const void *) src1_vtcm_data : src1_vtcm; + compute(src0_vtcm, w, dst_vtcm, block_size, uctx); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); + + const size_t dst_off = dst_contig ? (ir * nb1) : + unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3); + dma_queue_push(dma_q, + dma_make_data(data_dst + dst_off, dst_vtcm), + nb1, dst_row_size_aligned, dst_data_row_size, block_size); + + const uint32_t next_ir = ir + block_size; + if (next_ir < src0_end_row) { + const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_src0_contig, + block_dst_contig, ne01, div_ne01); + const uint32_t pref_ir = next_ir + next_block_size; + if (pref_ir < src0_end_row) { + const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_src0_contig, + block_dst_contig, ne01, div_ne01); + const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) : + unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm, data_src + src0_pref_off), + src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); + + if (!uctx->broadcast_weight) { + const size_t src1_pref_off = src1_contig ? (pref_ir * nb11) : + unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, + nb13_bc); + dma_queue_push(dma_q, + dma_make_data(src1_vtcm, data_src1 + src1_pref_off), + uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, pref_block_size); + } + } + } + ir += block_size; + } + + dma_queue_flush(dma_q); +} + +// 3. TRI row-block task with row index ir. +static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const size_t src0_data_row_size = uctx->src0_data_row_size; + const size_t dst_data_row_size = uctx->dst_data_row_size; + const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; + const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_dst = uctx->data_dst; + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + + const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; + const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; + + const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && + (nb03 == (size_t)ne02 * nb02); + const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && + (nb3 == (size_t)ne2 * nb2); + + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; + const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; + + const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); + const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); + const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); + if (BLOCK == 0) { + FARF(ERROR, "unary-tri : current VTCM reservation %zu is too small, needed at least %zu\n", + uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); + return; + } + + dma_queue * dma_q = octx->ctx->dma[ith]; + + for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, + ne01, div_ne01); + + dma_queue_push(dma_q, + dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), + nb1, dst_row_size_aligned, dst_data_row_size, 0); + + const size_t src0_off = src0_contig ? (ir * nb01) : + unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), + src0_row_size_aligned, nb01, src0_data_row_size, block_size); + + ir += block_size; + } + + unary_tri_compute_fn_t compute = (unary_tri_compute_fn_t) uctx->compute; + + for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, + ne01, div_ne01); + + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); + compute(src0_vtcm, dst_vtcm, block_size, ir, uctx); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); + + const size_t dst_off = dst_contig ? (ir * nb1) : + unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3); + dma_queue_push(dma_q, + dma_make_data(data_dst + dst_off, dst_vtcm), + nb1, dst_row_size_aligned, dst_data_row_size, block_size); + + const uint32_t next_ir = ir + block_size; + if (next_ir < src0_end_row) { + const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig, + dst_contig, ne01, div_ne01); + const uint32_t pref_ir = next_ir + next_block_size; + if (pref_ir < src0_end_row) { + const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig, + dst_contig, ne01, div_ne01); + const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) : + unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm, data_src + src0_pref_off), + src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); + } + } + ir += block_size; + } + + dma_queue_flush(dma_q); +} + +// 4. Pointwise tiled unary task. +static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t col_tile = uctx->col_tile; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_dst = uctx->data_dst; + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + + const size_t src0_half = uctx->src0_vtcm_half_size; + const size_t dst_half = uctx->dst_vtcm_half_size; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; + const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; + const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; + + const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; + + const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && + (nb03 == (size_t)ne02 * nb02); + const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && + (nb3 == (size_t)ne2 * nb2); + + const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; + + for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { + const uint32_t row = src0_start_row + t / tiles_per_row; + const uint32_t col = (t % tiles_per_row) * col_tile; + const uint32_t tw = MIN(col_tile, ne0 - col); + const size_t tb = (size_t) tw * sizeof(float); + const size_t soff = (src0_contig ? (row * nb01) : + unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) + + (size_t) col * sizeof(float); + + dma_queue_push(dma_q, dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); + dma_queue_push(dma_q, dma_make_data(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1); + } + + unary_tile_compute_fn_t compute = (unary_tile_compute_fn_t) uctx->compute; + + uint32_t row = src0_start_row; + uint32_t col = 0; + uint32_t tile_in_row = 0; + + uint32_t prow = src0_start_row + fastdiv(2, div_tpr); + uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; + uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); + + for (uint32_t t = 0; t < total_tiles; t++) { + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + const uint32_t tw = MIN(col_tile, ne0 - col); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); + compute(dst_vtcm, src_vtcm, tw, uctx); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); + + const size_t doff = (dst_contig ? (row * nb1) : + unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + + (size_t) col * sizeof(float); + const size_t tb = (size_t) tw * sizeof(float); + dma_queue_push(dma_q, dma_make_data(data_dst + doff, dst_vtcm), tb, tb, tb, 1); + + const uint32_t pt = t + 2; + if (pt < total_tiles) { + const uint32_t ptw = MIN(col_tile, ne0 - pcol); + const size_t ptb = (size_t) ptw * sizeof(float); + const size_t psoff = (src0_contig ? (prow * nb01) : + unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, + nb03)) + + (size_t) pcol * sizeof(float); + dma_queue_push(dma_q, dma_make_data(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); + } + + tile_in_row++; + col += col_tile; + if (tile_in_row == tiles_per_row) { + tile_in_row = 0; + col = 0; + row++; + } + + ptile_in_row++; + pcol += col_tile; + if (ptile_in_row == tiles_per_row) { + ptile_in_row = 0; + pcol = 0; + prow++; + } + } + + dma_queue_flush(dma_q); +} + +// 5. TRI tiled task. +static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const int32_t * op_params = octx->op_params; + const uint32_t col_tile = uctx->col_tile; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_dst = uctx->data_dst; + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + + const size_t src0_half = uctx->src0_vtcm_half_size; + const size_t dst_half = uctx->dst_vtcm_half_size; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; + const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; + const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; + + const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; + const int32_t tri_ttype = op_params[0]; + + const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && + (nb03 == (size_t)ne02 * nb02); + const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && + (nb3 == (size_t)ne2 * nb2); + + const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; + + for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { + const uint32_t row = src0_start_row + t / tiles_per_row; + const uint32_t col = (t % tiles_per_row) * col_tile; + const uint32_t tw = MIN(col_tile, ne0 - col); + const size_t tb = (size_t) tw * sizeof(float); + const size_t soff = (src0_contig ? (row * nb01) : + unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) + + (size_t) col * sizeof(float); + + dma_queue_push(dma_q, dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); + dma_queue_push(dma_q, dma_make_data(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1); + } + + unary_tiled_tri_compute_fn_t compute = (unary_tiled_tri_compute_fn_t) uctx->compute; + + uint32_t row = src0_start_row; + uint32_t col = 0; + uint32_t tile_in_row = 0; + uint32_t i01 = fastmodulo(row, ne01, div_ne01); + + uint32_t prow = src0_start_row + fastdiv(2, div_tpr); + uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; + uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); + + for (uint32_t t = 0; t < total_tiles; t++) { + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + const uint32_t tw = MIN(col_tile, ne0 - col); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); + compute(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); + + const size_t doff = (dst_contig ? (row * nb1) : + unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + + (size_t) col * sizeof(float); + const size_t tb = (size_t) tw * sizeof(float); + dma_queue_push(dma_q, dma_make_data(data_dst + doff, dst_vtcm), tb, tb, tb, 1); + + const uint32_t pt = t + 2; + if (pt < total_tiles) { + const uint32_t ptw = MIN(col_tile, ne0 - pcol); + const size_t ptb = (size_t) ptw * sizeof(float); + const size_t psoff = (src0_contig ? (prow * nb01) : + unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, + nb03)) + + (size_t) pcol * sizeof(float); + dma_queue_push(dma_q, dma_make_data(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); + } + + tile_in_row++; + col += col_tile; + if (tile_in_row == tiles_per_row) { + tile_in_row = 0; + col = 0; + row++; + i01++; + if (i01 == ne01) { + i01 = 0; + } + } + + ptile_in_row++; + pcol += col_tile; + if (ptile_in_row == tiles_per_row) { + ptile_in_row = 0; + pcol = 0; + prow++; + } + } + + dma_queue_flush(dma_q); +} static int execute_op_unary(struct htp_ops_context * octx) { int err = HTP_STATUS_OK; @@ -1064,6 +1473,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break; case HTP_OP_SCALE: op_type = is_f16 ? "scale-f16" : "scale-f32"; break; case HTP_OP_CLAMP: op_type = is_f16 ? "clamp-f16" : "clamp-f32"; break; + case HTP_OP_LEAKY_RELU: op_type = "leaky-relu-f32"; break; case HTP_OP_SQR: op_type = is_f16 ? "sqr-f16" : "sqr-f32"; break; case HTP_OP_SQRT: op_type = is_f16 ? "sqrt-f16" : "sqrt-f32"; break; case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break; @@ -1075,9 +1485,9 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break; case HTP_OP_UNARY_ABS: op_type = is_f16 ? "abs-f16" : "abs-f32"; break; case HTP_OP_UNARY_LOG: op_type = is_f16 ? "log-f16" : "log-f32"; break; + case HTP_OP_UNARY_RELU: op_type = "relu-f32"; break; case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break; case HTP_OP_TRI: op_type = "tri-f32"; break; - default: FARF(ERROR, "Unsupported unary Op %u\n", octx->op); return HTP_STATUS_NO_SUPPORT; @@ -1105,14 +1515,32 @@ static int execute_op_unary(struct htp_ops_context * octx) { const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params; - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = kparams->n_threads; + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float); - const size_t src0_data_row_size = src0->ne[0] * elem_size; const size_t dst_data_row_size = dst->ne[0] * elem_size; + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + const size_t src0_row_size_aligned = kparams->src0_row_size_aligned; const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; @@ -1145,110 +1573,128 @@ static int execute_op_unary(struct htp_ops_context * octx) { src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], kparams->vtcm_src0_size, kparams->vtcm_src1_size, kparams->vtcm_dst_size); - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; - struct htp_unary_context uctx = { - .octx = octx, - .kparams = kparams, - .src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads, - .src0_nrows = src0_nrows, - - .data_src0 = (const uint8_t *)src0->data, - .data_src1 = (octx->op == HTP_OP_RMS_NORM_MUL) ? (const uint8_t *)src1->data : NULL, - .data_dst = (uint8_t *)dst->data, - - .src0_data_row_size = src0_data_row_size, - .src1_data_row_size = src1_data_row_size, - .dst_data_row_size = dst_data_row_size, - - .src0_row_size_aligned = src0_row_size_aligned, - .src1_row_size_aligned = src1_row_size_aligned, - .dst_row_size_aligned = dst_row_size_aligned, - - .src0_vtcm_half_size = kparams->vtcm_src0_size_per_thread / 2, - .src1_vtcm_half_size = (octx->op == HTP_OP_RMS_NORM_MUL) ? (kparams->vtcm_src1_size_per_thread / (broadcast_weight ? 1 : 2)) : 0, - .dst_vtcm_half_size = kparams->vtcm_dst_size_per_thread / 2, - - .block = kparams->block, - .nc = src0->ne[0], - .col_tile = col_tile, - .broadcast_weight = broadcast_weight, - - .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, 0), - .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, kparams->vtcm_src0_size, kparams->vtcm_src1_size > 0), - .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, kparams->vtcm_src0_size + kparams->vtcm_src1_size), - - .vtcm_src0_size_per_thread = kparams->vtcm_src0_size_per_thread, - .vtcm_src1_size_per_thread = kparams->vtcm_src1_size_per_thread, - .vtcm_dst_size_per_thread = kparams->vtcm_dst_size_per_thread, - }; - - FARF(HIGH, "%s: %s mode (col_tile %u)\n", op_type, col_tile ? "tiled" : "row-block", col_tile); - - worker_callback_t task_func = NULL; - if (col_tile) { - switch (octx->op) { - case HTP_OP_SCALE: task_func = unary_task_f32_tiled_scale; break; - case HTP_OP_CLAMP: task_func = unary_task_f32_tiled_clamp; break; - case HTP_OP_SQR: task_func = unary_task_f32_tiled_sqr; break; - case HTP_OP_SQRT: task_func = unary_task_f32_tiled_sqrt; break; - case HTP_OP_UNARY_NEG: task_func = unary_task_f32_tiled_unary_neg; break; - case HTP_OP_UNARY_EXP: task_func = unary_task_f32_tiled_unary_exp; break; - case HTP_OP_UNARY_SIGMOID: task_func = unary_task_f32_tiled_unary_sigmoid; break; - case HTP_OP_UNARY_SILU: task_func = unary_task_f32_tiled_unary_silu; break; - case HTP_OP_UNARY_GELU: task_func = unary_task_f32_tiled_unary_gelu; break; - case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_tiled_unary_softplus; break; - case HTP_OP_UNARY_TANH: task_func = unary_task_f32_tiled_unary_tanh; break; - case HTP_OP_UNARY_ABS: task_func = unary_task_f32_tiled_unary_abs; break; - case HTP_OP_UNARY_LOG: task_func = unary_task_f32_tiled_unary_log; break; - case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break; - default: break; - } - } else if (is_f16) { - switch (octx->op) { - case HTP_OP_NORM: task_func = unary_task_f16_norm; break; - case HTP_OP_RMS_NORM: task_func = unary_task_f16_rms_norm; break; - case HTP_OP_SCALE: task_func = unary_task_f16_scale; break; - case HTP_OP_CLAMP: task_func = unary_task_f16_clamp; break; - case HTP_OP_SQR: task_func = unary_task_f16_sqr; break; - case HTP_OP_SQRT: task_func = unary_task_f16_sqrt; break; - case HTP_OP_L2_NORM: task_func = unary_task_f16_l2_norm; break; - case HTP_OP_UNARY_ABS: task_func = unary_task_f16_unary_abs; break; - case HTP_OP_UNARY_LOG: task_func = unary_task_f16_unary_log; break; - default: break; - } - } else { - switch (octx->op) { - case HTP_OP_NORM: task_func = unary_task_f32_norm; break; - case HTP_OP_RMS_NORM: task_func = unary_task_f32_rms_norm; break; - case HTP_OP_RMS_NORM_MUL: task_func = unary_task_f32_rms_norm_mul; break; - case HTP_OP_SCALE: task_func = unary_task_f32_scale; break; - case HTP_OP_CLAMP: task_func = unary_task_f32_clamp; break; - case HTP_OP_SQR: task_func = unary_task_f32_sqr; break; - case HTP_OP_SQRT: task_func = unary_task_f32_sqrt; break; - case HTP_OP_UNARY_NEG: task_func = unary_task_f32_unary_neg; break; - case HTP_OP_UNARY_EXP: task_func = unary_task_f32_unary_exp; break; - case HTP_OP_UNARY_SIGMOID: task_func = unary_task_f32_unary_sigmoid; break; - case HTP_OP_UNARY_SILU: task_func = unary_task_f32_unary_silu; break; - case HTP_OP_UNARY_GELU: task_func = unary_task_f32_unary_gelu; break; - case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_unary_softplus; break; - case HTP_OP_UNARY_TANH: task_func = unary_task_f32_unary_tanh; break; - case HTP_OP_UNARY_ABS: task_func = unary_task_f32_unary_abs; break; - case HTP_OP_UNARY_LOG: task_func = unary_task_f32_unary_log; break; - case HTP_OP_L2_NORM: task_func = unary_task_f32_l2_norm; break; - case HTP_OP_TRI: task_func = unary_task_f32_tri; break; - default: break; - } - } + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + struct htp_unary_context uctx = { + .octx = octx, + .kparams = kparams, + .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .src0_nrows = nrows, + .row_start = row_start, - if (task_func) { - worker_pool_run_func(octx->ctx->worker_pool, task_func, &uctx, n_threads); - } else { - FARF(ERROR, "execute_op_unary: task function is NULL for op %d\n", octx->op); - err = HTP_STATUS_NO_SUPPORT; + .data_src0 = src0->data, + .data_src1 = (octx->op == HTP_OP_RMS_NORM_MUL) ? src1->data : 0, + .data_dst = dst->data, + + .src0_data_row_size = src0_data_row_size, + .src1_data_row_size = src1_data_row_size, + .dst_data_row_size = dst_data_row_size, + + .src0_row_size_aligned = src0_row_size_aligned, + .src1_row_size_aligned = src1_row_size_aligned, + .dst_row_size_aligned = dst_row_size_aligned, + + .src0_vtcm_half_size = kparams->vtcm_src0_size_per_thread / 2, + .src1_vtcm_half_size = (octx->op == HTP_OP_RMS_NORM_MUL) ? (kparams->vtcm_src1_size_per_thread / (broadcast_weight ? 1 : 2)) : 0, + .dst_vtcm_half_size = kparams->vtcm_dst_size_per_thread / 2, + + .block = kparams->block, + .nc = src0->ne[0], + .col_tile = col_tile, + .broadcast_weight = broadcast_weight, + + .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, 0), + .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, kparams->vtcm_src0_size, kparams->vtcm_src1_size > 0), + .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, kparams->vtcm_src0_size + kparams->vtcm_src1_size), + + .vtcm_src0_size_per_thread = kparams->vtcm_src0_size_per_thread, + .vtcm_src1_size_per_thread = kparams->vtcm_src1_size_per_thread, + .vtcm_dst_size_per_thread = kparams->vtcm_dst_size_per_thread, + }; + + FARF(HIGH, "%s: %s mode (col_tile %u)\n", op_type, col_tile ? "tiled" : "row-block", col_tile); + + worker_callback_t task_func = NULL; + void * compute_func = NULL; + + if (col_tile) { + task_func = unary_thread_tiled; + switch (octx->op) { + case HTP_OP_SCALE: compute_func = (void *) tile_scale_f32; break; + case HTP_OP_CLAMP: compute_func = (void *) tile_clamp_f32; break; + case HTP_OP_LEAKY_RELU: compute_func = (void *) tile_leaky_relu_f32; break; + case HTP_OP_SQR: compute_func = (void *) tile_sqr_f32; break; + case HTP_OP_SQRT: compute_func = (void *) tile_sqrt_f32; break; + case HTP_OP_UNARY_NEG: compute_func = (void *) tile_neg_f32; break; + case HTP_OP_UNARY_EXP: compute_func = (void *) tile_exp_f32; break; + case HTP_OP_UNARY_SIGMOID: compute_func = (void *) tile_sigmoid_f32; break; + case HTP_OP_UNARY_SILU: compute_func = (void *) tile_silu_f32; break; + case HTP_OP_UNARY_GELU: compute_func = (void *) tile_gelu_f32; break; + case HTP_OP_UNARY_SOFTPLUS: compute_func = (void *) tile_softplus_f32; break; + case HTP_OP_UNARY_TANH: compute_func = (void *) tile_tanh_f32; break; + case HTP_OP_UNARY_ABS: compute_func = (void *) tile_abs_f32; break; + case HTP_OP_UNARY_LOG: compute_func = (void *) tile_log_f32; break; + case HTP_OP_UNARY_RELU: compute_func = (void *) tile_relu_f32; break; + case HTP_OP_TRI: + task_func = unary_thread_tiled_tri_f32; + compute_func = (void *) tri_apply_tile_f32; + break; + default: break; + } + } else if (is_f16) { + task_func = unary_thread_row_block; + switch (octx->op) { + case HTP_OP_NORM: compute_func = (void *) norm_f16; break; + case HTP_OP_RMS_NORM: compute_func = (void *) rms_norm_f16; break; + case HTP_OP_SCALE: compute_func = (void *) scale_f16; break; + case HTP_OP_CLAMP: compute_func = (void *) clamp_f16; break; + case HTP_OP_SQR: compute_func = (void *) sqr_f16; break; + case HTP_OP_SQRT: compute_func = (void *) sqrt_f16; break; + case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f16; break; + case HTP_OP_UNARY_ABS: compute_func = (void *) abs_f16; break; + case HTP_OP_UNARY_LOG: compute_func = (void *) log_f16; break; + default: break; + } + } else { + task_func = unary_thread_row_block; + switch (octx->op) { + case HTP_OP_NORM: compute_func = (void *) norm_f32; break; + case HTP_OP_RMS_NORM: compute_func = (void *) rms_norm_f32; break; + case HTP_OP_RMS_NORM_MUL: + task_func = unary_thread_rms_norm_mul_f32; + compute_func = (void *) rms_norm_mul_f32; + break; + case HTP_OP_SCALE: compute_func = (void *) scale_f32; break; + case HTP_OP_CLAMP: compute_func = (void *) clamp_f32; break; + case HTP_OP_LEAKY_RELU: compute_func = (void *) leaky_relu_f32; break; + case HTP_OP_SQR: compute_func = (void *) sqr_f32; break; + case HTP_OP_SQRT: compute_func = (void *) sqrt_f32; break; + case HTP_OP_UNARY_NEG: compute_func = (void *) neg_f32; break; + case HTP_OP_UNARY_EXP: compute_func = (void *) exp_f32; break; + case HTP_OP_UNARY_SIGMOID: compute_func = (void *) sigmoid_f32; break; + case HTP_OP_UNARY_SILU: compute_func = (void *) silu_f32; break; + case HTP_OP_UNARY_GELU: compute_func = (void *) gelu_f32; break; + case HTP_OP_UNARY_SOFTPLUS: compute_func = (void *) softplus_f32; break; + case HTP_OP_UNARY_TANH: compute_func = (void *) tanh_f32; break; + case HTP_OP_UNARY_ABS: compute_func = (void *) abs_f32; break; + case HTP_OP_UNARY_LOG: compute_func = (void *) log_f32; break; + case HTP_OP_UNARY_RELU: compute_func = (void *) relu_f32; break; + case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f32; break; + case HTP_OP_TRI: + task_func = unary_thread_tri_f32; + compute_func = (void *) tri_f32; + break; + default: break; } } + if (!task_func || !compute_func) { + FARF(ERROR, "execute_op_unary: task function is NULL for op %d\n", octx->op); + return HTP_STATUS_NO_SUPPORT; + } + + uctx.compute = compute_func; + work_queue_run(octx->ctx->work_queue, task_func, &uctx, n_threads); + return err; } diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h index 116a591c2d77..e410d7fd83aa 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.h +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -42,6 +42,7 @@ _Static_assert(sizeof(struct htp_unary_kernel_params) <= 128, "htp_unary_kernel_ static inline bool htp_op_is_unary(uint32_t opcode) { switch (opcode) { case HTP_OP_CLAMP: + case HTP_OP_LEAKY_RELU: case HTP_OP_NORM: case HTP_OP_RMS_NORM: case HTP_OP_RMS_NORM_MUL: @@ -57,6 +58,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) { case HTP_OP_UNARY_TANH: case HTP_OP_UNARY_ABS: case HTP_OP_UNARY_LOG: + case HTP_OP_UNARY_RELU: case HTP_OP_L2_NORM: case HTP_OP_TRI: return true; diff --git a/ggml/src/ggml-hip/CMakeLists.txt b/ggml/src/ggml-hip/CMakeLists.txt index f29996530baa..b5f392ccce07 100644 --- a/ggml/src/ggml-hip/CMakeLists.txt +++ b/ggml/src/ggml-hip/CMakeLists.txt @@ -237,32 +237,8 @@ list(APPEND GGML_SOURCES_ROCM ${SRCS}) file(GLOB SRCS "../ggml-cuda/template-instances/mmf*.cu") list(APPEND GGML_SOURCES_ROCM ${SRCS}) -if (GGML_CUDA_FA_ALL_QUANTS) - file(GLOB SRCS "../ggml-cuda/template-instances/fattn-vec*.cu") - list(APPEND GGML_SOURCES_ROCM ${SRCS}) - add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS) -else() - list(APPEND GGML_SOURCES_ROCM - ../ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-q8_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-q8_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo2_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo3_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo4_0-turbo4_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo4_0-q8_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo4_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo4_0-turbo3_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo4_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo4_0-turbo2_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo4_0.cu) -endif() +ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR}/../ggml-cuda SRCS) +list(APPEND GGML_SOURCES_ROCM ${SRCS}) ggml_add_backend_library(ggml-hip ${GGML_HEADERS_ROCM} diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h index 62b76abbcec9..ae26e0c23b46 100644 --- a/ggml/src/ggml-impl.h +++ b/ggml/src/ggml-impl.h @@ -160,6 +160,18 @@ static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t return ((const float *)(tensor->op_params))[i]; } +// [TAG_GGML_PREC] +// - GGML_OP_MUL_MAT +// 0 - acc +// 1 - hint +// 2 - src0 precision +// 3 - src1 precision +// +// - GGML_OP_MUL_MAT_ID +// 0 - acc +// 1 - hint +// 2 - src0 precision +// 3 - src1 precision static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) { assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t)); ((int32_t *)(tensor->op_params))[i] = value; diff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txt index a661e710a2f2..e7afdb69572f 100644 --- a/ggml/src/ggml-metal/CMakeLists.txt +++ b/ggml/src/ggml-metal/CMakeLists.txt @@ -10,6 +10,7 @@ ggml_add_backend_library(ggml-metal ggml-metal-device.cpp ggml-metal-common.cpp ggml-metal-context.m + ggml-metal-fusion.cpp ggml-metal-ops.cpp ggml-metal-tuning.cpp ) diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cpp index 6f1638a1147e..9c0b9474c256 100644 --- a/ggml/src/ggml-metal/ggml-metal-common.cpp +++ b/ggml/src/ggml-metal/ggml-metal-common.cpp @@ -1,4 +1,5 @@ #include "ggml-metal-common.h" +#include "ggml-metal-fusion.h" #include "ggml.h" #include "ggml-impl.h" @@ -6,6 +7,24 @@ #include <vector> +// must stay in sync with the kernel_fwht_<type>_<N> templates in misc.metal +static bool ggml_metal_fwht_supported_size(int64_t n) { + return n == 64 || n == 128 || n == 256 || n == 512; +} + +// the FWHT kernels handle a Hadamard-hinted MUL_MAT only under these conditions. supports_op +// and the dispatch must ask the same question: an F16 src1 that is admitted but then falls +// through reaches the generic path, which has no F32 src0 by F16 src1 kernel. +bool ggml_metal_op_mul_mat_use_fwht(const struct ggml_tensor * op) { + return ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD && + op->type == GGML_TYPE_F32 && + (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) && + ggml_is_contiguous(op->src[1]) && + ggml_is_contiguous(op) && + ggml_are_same_shape(op->src[1], op) && + ggml_metal_fwht_supported_size(op->src[1]->ne[0]); +} + bool ggml_metal_op_mul_mat_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) { const int64_t ne00 = op->src[0]->ne[0]; const int64_t ne11 = op->src[1]->ne[1]; @@ -221,38 +240,63 @@ struct node_info { void add_fused(ggml_tensor * t) { fused.push_back(t); } + + bool is_output(const ggml_tensor * t) const { + if (t == node) { + return true; + } + for (const auto * f : fused) { + if (t == f) { + return true; + } + } + return false; + } }; static std::vector<int> ggml_metal_graph_optimize_reorder(const std::vector<node_info> & nodes) { // helper to add node src and dst ranges const auto & h_add = [](ggml_mem_ranges_t mrs, const node_info & node) { + // only external sources matter: sources produced by the fused group are internal for (int i = 0; i < GGML_MAX_SRC; i++) { - if (node.node->src[i]) { - if (!ggml_mem_ranges_add_src(mrs, node.node->src[i])) { + const ggml_tensor * src = node.node->src[i]; + if (src && !node.is_output(src)) { + if (!ggml_mem_ranges_add_src(mrs, src)) { return false; } } } - // keep track of the sources of the fused nodes as well for (const auto * fused : node.fused) { for (int i = 0; i < GGML_MAX_SRC; i++) { - if (fused->src[i]) { - if (!ggml_mem_ranges_add_src(mrs, fused->src[i])) { + const ggml_tensor * src = fused->src[i]; + if (src && !node.is_output(src)) { + if (!ggml_mem_ranges_add_src(mrs, src)) { return false; } } } } - return ggml_mem_ranges_add_dst(mrs, node.dst()); + // all fused tensors are produced by the fused kernel + if (!ggml_mem_ranges_add_dst(mrs, node.node)) { + return false; + } + for (const auto * fused : node.fused) { + if (!ggml_mem_ranges_add_dst(mrs, fused)) { + return false; + } + } + + return true; }; // helper to check if a node can run concurrently with the existing set of nodes const auto & h_check = [](ggml_mem_ranges_t mrs, const node_info & node) { for (int i = 0; i < GGML_MAX_SRC; i++) { - if (node.node->src[i]) { - if (!ggml_mem_ranges_check_src(mrs, node.node->src[i])) { + const ggml_tensor * src = node.node->src[i]; + if (src && !node.is_output(src)) { + if (!ggml_mem_ranges_check_src(mrs, src)) { return false; } } @@ -260,15 +304,25 @@ static std::vector<int> ggml_metal_graph_optimize_reorder(const std::vector<node for (const auto * fused : node.fused) { for (int i = 0; i < GGML_MAX_SRC; i++) { - if (fused->src[i]) { - if (!ggml_mem_ranges_check_src(mrs, fused->src[i])) { + const ggml_tensor * src = fused->src[i]; + if (src && !node.is_output(src)) { + if (!ggml_mem_ranges_check_src(mrs, src)) { return false; } } } } - return ggml_mem_ranges_check_dst(mrs, node.dst()); + if (!ggml_mem_ranges_check_dst(mrs, node.node)) { + return false; + } + for (const auto * fused : node.fused) { + if (!ggml_mem_ranges_check_dst(mrs, fused)) { + return false; + } + } + + return true; }; // perform reorders only across these types of ops @@ -390,59 +444,31 @@ static std::vector<int> ggml_metal_graph_optimize_reorder(const std::vector<node } void ggml_graph_optimize(ggml_cgraph * gf) { - constexpr int MAX_FUSE = 16; - const int n = gf->n_nodes; - enum ggml_op ops[MAX_FUSE]; - std::vector<node_info> nodes; nodes.reserve(gf->n_nodes); // fuse nodes: // we don't want to make reorders that break fusing, so we first pack all fusable tensors // and perform the reorder over the fused nodes. after the reorder is done, we unfuse + // + // the fusable sequences are declared in the fusion table (ggml-metal-fuse.cpp), so the + // packing here is driven by the same patterns that the op encoders will later use for (int i = 0; i < n; i++) { node_info node = { /*.node =*/ gf->nodes[i], /*.fused =*/ {}, }; - // fuse only ops that start with these operations - // can be expanded when needed - if (node.op() == GGML_OP_ADD || - node.op() == GGML_OP_NORM || - node.op() == GGML_OP_RMS_NORM) { - ops[0] = node.op(); - - int f = i + 1; - while (f < n && f < i + MAX_FUSE) { - // conservatively allow fusing only these ops - // can be expanded when needed - if (gf->nodes[f]->op != GGML_OP_ADD && - gf->nodes[f]->op != GGML_OP_MUL && - gf->nodes[f]->op != GGML_OP_NORM && - gf->nodes[f]->op != GGML_OP_RMS_NORM) { - break; - } - ops[f - i] = gf->nodes[f]->op; - f++; - } - - f -= i; - for (; f > 1; f--) { - if (ggml_can_fuse(gf, i, ops, f)) { - break; - } - } + const int f = ggml_metal_fusion_max(gf, i); - // add the fused tensors into the node info so we can unfuse them later - for (int k = 1; k < f; k++) { - ++i; + // add the fused tensors into the node info so we can unfuse them later + for (int k = 1; k < f; k++) { + ++i; - // the .dst() becomes the last fused tensor - node.add_fused(gf->nodes[i]); - } + // the .dst() becomes the last fused tensor + node.add_fused(gf->nodes[i]); } nodes.push_back(std::move(node)); diff --git a/ggml/src/ggml-metal/ggml-metal-common.h b/ggml/src/ggml-metal/ggml-metal-common.h index 66abdb52efe3..e6a28d032d78 100644 --- a/ggml/src/ggml-metal/ggml-metal-common.h +++ b/ggml/src/ggml-metal/ggml-metal-common.h @@ -48,6 +48,7 @@ bool ggml_mem_ranges_check(ggml_mem_ranges_t mrs, const struct ggml_tensor * ten void ggml_graph_optimize(struct ggml_cgraph * gf); // mat-mat vs mat-vec dispatch; used by both supports_op and ggml_metal_op_mul_mat* +bool ggml_metal_op_mul_mat_use_fwht (const struct ggml_tensor * op); bool ggml_metal_op_mul_mat_use_mm (const struct ggml_tensor * op, bool has_simdgroup_mm); bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm); diff --git a/ggml/src/ggml-metal/ggml-metal-context.h b/ggml/src/ggml-metal/ggml-metal-context.h index abf4b06ed2ab..b538b1ad20a1 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.h +++ b/ggml/src/ggml-metal/ggml-metal-context.h @@ -33,6 +33,7 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx); void ggml_metal_set_n_cb (ggml_metal_t ctx, int n_cb); void ggml_metal_set_abort_callback (ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data); + bool ggml_metal_supports_family (ggml_metal_t ctx, int family); void ggml_metal_capture_next_compute(ggml_metal_t ctx); diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index e1129db30212..442ed2a0745b 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -6,6 +6,7 @@ #import "ggml-metal-impl.h" #import "ggml-metal-common.h" #import "ggml-metal-ops.h" +#import "ggml-metal-fusion.h" #import <Foundation/Foundation.h> @@ -29,22 +30,20 @@ ggml_metal_device_t dev; ggml_metal_library_t lib; - ggml_metal_event_t ev_cpy; // for async copies + ggml_metal_event_t ev_cpy; // for async copies + ggml_metal_event_t ev_sync; // destination completion signal dispatch_queue_t d_queue; // additional, inference-time compiled pipelines ggml_metal_pipelines_t pipelines_ext; - bool use_fusion; bool use_concurrency; bool use_graph_optimize; int debug_graph; - int debug_fusion; - // how many times a given op was fused - uint64_t fuse_cnt[GGML_OP_COUNT]; + struct ggml_metal_fusion_info * finfo; // capture state int capture_compute; @@ -111,6 +110,7 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { id<MTLCommandQueue> queue = ggml_metal_device_get_queue(dev); if (queue == nil) { GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__); + free(res); return NULL; } @@ -130,7 +130,8 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { } } - res->ev_cpy = ggml_metal_device_event_init(dev); + res->ev_cpy = ggml_metal_device_event_init(dev); + res->ev_sync = ggml_metal_device_event_init(dev); const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev); @@ -138,7 +139,6 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT); - res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil; res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil; { @@ -146,20 +146,19 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { res->debug_graph = val ? atoi(val) : 0; } - { - const char * val = getenv("GGML_METAL_FUSION_DEBUG"); - res->debug_fusion = val ? atoi(val) : 0; - } - res->use_graph_optimize = true; if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) { res->use_graph_optimize = false; } - memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt)); + res->finfo = ggml_metal_device_get_fusion_info(dev); + if (ggml_metal_fusion_info_stats(res->finfo)) { + ggml_metal_fusion_info_labels_init(res->finfo); + res->n_cb = 0; + } - GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false"); + GGML_LOG_INFO("%s: use fusion = %s\n", __func__, ggml_metal_fusion_info_enabled(res->finfo) ? "true" : "false"); GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false"); GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false"); @@ -221,15 +220,18 @@ void ggml_metal_free(ggml_metal_t ctx) { ctx->pipelines_ext = nil; } - if (ctx->debug_fusion > 0) { + if (ggml_metal_fusion_info_debug(ctx->finfo) > 0) { GGML_LOG_DEBUG("%s: fusion stats:\n", __func__); - for (int i = 0; i < GGML_OP_COUNT; i++) { - if (ctx->fuse_cnt[i] == 0) { + + const int n_fusions = ggml_metal_fusion_info_n_fusions(ctx->finfo); + for (int i = 0; i < n_fusions; i++) { + const uint64_t count = ggml_metal_fusion_info_count(ctx->finfo, i); + if (count == 0) { continue; } // note: cannot use ggml_log here - GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_op_name((enum ggml_op) i), ctx->fuse_cnt[i]); + GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_metal_fusion_info_label(ctx->finfo, i), count); } } @@ -240,6 +242,7 @@ void ggml_metal_free(ggml_metal_t ctx) { dispatch_release(ctx->d_queue); ggml_metal_device_event_free(ctx->dev, ctx->ev_cpy); + ggml_metal_device_event_free(ctx->dev, ctx->ev_sync); free(ctx); } @@ -421,10 +424,23 @@ bool ggml_metal_cpy_tensor_async(ggml_metal_t ctx_src, ggml_metal_t ctx_dst, con return false; } + id<MTLCommandQueue> dst_queue = ggml_metal_device_get_queue(ctx_dst->dev); + id<MTLCommandBuffer> sync_cmd_buf = [dst_queue commandBuffer]; + + ggml_metal_event_encode_signal(ctx_dst->ev_sync, sync_cmd_buf); + + [sync_cmd_buf commit]; + + [ctx_dst->cmd_bufs_ext addObject:sync_cmd_buf]; + ctx_dst->cmd_buf_last = sync_cmd_buf; + + [sync_cmd_buf retain]; + // queue the copy operation into the Metal context // this will be queued at the end, after any currently ongoing GPU operations id<MTLCommandQueue> queue = ggml_metal_device_get_queue(ctx_src->dev); id<MTLCommandBuffer> cmd_buf = [queue commandBuffer]; + ggml_metal_event_encode_wait(ctx_dst->ev_sync, cmd_buf); id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder]; [encoder copyFromBuffer:bid_src.metal @@ -480,10 +496,17 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * @autoreleasepool { ctx->gf = gf; - ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); - ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; + if (ctx->n_cb == 0) { + // single-threaded encoding: the whole graph is encoded by one command buffer + ctx->n_nodes_0 = gf->n_nodes; + ctx->n_nodes_1 = 0; + ctx->n_nodes_per_cb = 0; + } else { + ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); + ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; - ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + } if (ctx->capture_compute >= 0) { ctx->capture_compute--; @@ -681,6 +704,12 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) { } void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { + // when fusion stats are collected the graph must be encoded by a single thread so the + // counters are race-free; override whatever the caller requested + if (ggml_metal_fusion_info_stats(ctx->finfo)) { + n_cb = 0; + } + if (ctx->n_cb != n_cb) { ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS); @@ -716,13 +745,12 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { ctx->dev, cmd_buf, ctx->gf, + ctx->finfo, idx_start, idx_end, - ctx->use_fusion, ctx->use_concurrency, ctx->capture_compute, - ctx->debug_graph, - ctx->debug_fusion); + ctx->debug_graph); for (int idx = 0; idx < ggml_metal_op_n_nodes(ctx_op); ++idx) { const int res = ggml_metal_op_encode(ctx_op, idx); diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index c296d17b1571..dc6b695eb02e 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -496,25 +496,48 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexe return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, ggml_op op) { - const char * name = nullptr; +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, const ggml_tensor * op) { + char name[256]; + const char * base = nullptr; - switch (op) { - case GGML_OP_DSV4_HC_COMB: name = "kernel_dsv4_hc_comb_f32"; break; - case GGML_OP_DSV4_HC_PRE: name = "kernel_dsv4_hc_pre_f32"; break; - case GGML_OP_DSV4_HC_POST: name = "kernel_dsv4_hc_post_f32"; break; - default: GGML_ABORT("fatal error"); + switch (op->op) { + case GGML_OP_DSV4_HC_COMB: + base = "kernel_dsv4_hc_comb_f32"; + snprintf(name, 256, "%s", base); + break; + case GGML_OP_DSV4_HC_PRE: + base = ggml_get_op_params_i32(op, 1) != 0 ? "kernel_dsv4_hc_pre_gated_f32" : "kernel_dsv4_hc_pre_f32"; + snprintf(name, 256, "%s_n_hc=%d", base, (int) op->src[0]->ne[1]); + break; + case GGML_OP_DSV4_HC_POST: + base = op->src[3] ? "kernel_dsv4_hc_post_f32" : "kernel_dsv4_hc_post_nocomb_f32"; + snprintf(name, 256, "%s", base); + break; + default: + GGML_ABORT("fatal error"); } ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { - res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr); + ggml_metal_cv_t cv = nullptr; + + if (op->op == GGML_OP_DSV4_HC_PRE) { + cv = ggml_metal_cv_init(); + ggml_metal_cv_set_int32(cv, (int32_t) op->src[0]->ne[1], FC_DSV4_HC + 0); + } + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + if (cv) { + ggml_metal_cv_free(cv); + } } return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv(ggml_metal_library_t lib, const ggml_tensor * op) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv( + ggml_metal_library_t lib, const ggml_tensor * op, int32_t nc, bool use_silu) { GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32); GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); @@ -531,17 +554,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv(ggml_me } snprintf(base, 256, "kernel_ssm_conv_%s_%s%s", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type), suffix); - snprintf(name, 256, "%s", base); + snprintf(name, 256, "%s_nc=%d_silu=%d", base, nc, use_silu ? 1 : 0); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { - res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_bool(cv, use_silu, FC_SSM_CONV + 1); + ggml_metal_cv_set_int32(cv, nc, FC_SSM_CONV + 2); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); } return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched(ggml_metal_library_t lib, const ggml_tensor * op, int ssm_conv_bs) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched( + ggml_metal_library_t lib, const ggml_tensor * op, int ssm_conv_bs, int32_t nc, bool use_silu) { GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32); GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); @@ -557,13 +587,15 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched } snprintf(base, 256, "kernel_ssm_conv_%s_%s_batched%s", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type), suffix); - snprintf(name, 256, "%s_ssm_conv_bs=%d", base, ssm_conv_bs); + snprintf(name, 256, "%s_ssm_conv_bs=%d_nc=%d_silu=%d", base, ssm_conv_bs, nc, use_silu ? 1 : 0); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { ggml_metal_cv_t cv = ggml_metal_cv_init(); ggml_metal_cv_set_int16(cv, ssm_conv_bs, FC_SSM_CONV + 0); + ggml_metal_cv_set_bool(cv, use_silu, FC_SSM_CONV + 1); + ggml_metal_cv_set_int32(cv, nc, FC_SSM_CONV + 2); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -839,6 +871,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta const char * suffix = ""; + bool split = false; + // use custom matrix x vector kernel switch (tsrc0) { case GGML_TYPE_F32: @@ -930,39 +964,82 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta nsg = N_SG_IQ2_XXS; nr0 = N_R0_IQ2_XXS; smem = 256*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_XS: { nsg = N_SG_IQ2_XS; nr0 = N_R0_IQ2_XS; smem = 512*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_XXS: { nsg = N_SG_IQ3_XXS; nr0 = N_R0_IQ3_XXS; smem = 256*4+128; + + // split the rows across threads when there are fewer than 32 chunks per row + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_S: { nsg = N_SG_IQ3_S; nr0 = N_R0_IQ3_S; smem = 512*4; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_S: { nsg = N_SG_IQ2_S; nr0 = N_R0_IQ2_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_S: { nsg = N_SG_IQ1_S; nr0 = N_R0_IQ1_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_M: { nsg = N_SG_IQ1_M; nr0 = N_R0_IQ1_M; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_M_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ4_NL: { @@ -993,7 +1070,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta const int16_t r3 = (int16_t) (ne13 / ne03); snprintf(base, 256, "kernel_mul_mv_%s_%s%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1), suffix); - snprintf(name, 256, "%s_nsg=%d_ne12=%d_r2=%d_r3=%d", base, nsg, ne12, r2, r3); + snprintf(name, 256, "%s_nsg=%d_ne12=%d_r2=%d_r3=%d_split=%d", base, nsg, ne12, r2, r3, split); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { @@ -1003,6 +1080,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta ggml_metal_cv_set_int16(cv, (int16_t) ne12, FC_MUL_MV + 2); ggml_metal_cv_set_int16(cv, r2, FC_MUL_MV + 3); ggml_metal_cv_set_int16(cv, r3, FC_MUL_MV + 4); + ggml_metal_cv_set_bool (cv, split, FC_MUL_MV + 5); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -1017,6 +1095,40 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax_part(ggml_metal_library_t lib) { + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_mul_mm_id_amax_part_f32"); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + res.smem = 32*sizeof(float); + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax(ggml_metal_library_t lib) { + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_mul_mm_id_amax_f32"); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + res.smem = 32*sizeof(float); + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(ggml_metal_library_t lib, int ne02, int ne20) { char base[256]; char name[256]; @@ -1044,14 +1156,18 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id(ggml_m const bool bc_inp = op->src[0]->ne[0] % 32 != 0; + // src1 prec [TAG_GGML_PREC] + const bool amax = ggml_get_op_params_i32(op, 3) == GGML_PREC_F32; + snprintf(base, 256, "kernel_mul_mm_id_%s_%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1)); - snprintf(name, 256, "%s_bci=%d", base, bc_inp); + snprintf(name, 256, "%s_bci=%d_amax=%d", base, bc_inp, amax); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { ggml_metal_cv_t cv = ggml_metal_cv_init(); ggml_metal_cv_set_bool(cv, bc_inp, FC_MUL_MM + 0); + ggml_metal_cv_set_bool(cv, amax, FC_MUL_MM + 6); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -1081,6 +1197,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m const char * suffix = ""; + bool split = false; + // use custom matrix x vector kernel switch (tsrc0) { case GGML_TYPE_F32: @@ -1165,39 +1283,82 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m nsg = N_SG_IQ2_XXS; nr0 = N_R0_IQ2_XXS; smem = 256*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_XS: { nsg = N_SG_IQ2_XS; nr0 = N_R0_IQ2_XS; smem = 512*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_XXS: { nsg = N_SG_IQ3_XXS; nr0 = N_R0_IQ3_XXS; smem = 256*4+128; + + // split the rows across threads when there are fewer than 32 chunks per row + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_S: { nsg = N_SG_IQ3_S; nr0 = N_R0_IQ3_S; smem = 512*4; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_S: { nsg = N_SG_IQ2_S; nr0 = N_R0_IQ2_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_S: { nsg = N_SG_IQ1_S; nr0 = N_R0_IQ1_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_M: { nsg = N_SG_IQ1_M; nr0 = N_R0_IQ1_M; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_M_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ4_NL: { @@ -1224,7 +1385,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m }; snprintf(base, 256, "kernel_mul_mv_id_%s_%s%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1), suffix); - snprintf(name, 256, "%s_nsg=%d", base, nsg); + snprintf(name, 256, "%s_nsg=%d_split=%d", base, nsg, split); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { @@ -1234,6 +1395,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m ggml_metal_cv_set_int16(cv, 1, FC_MUL_MV + 2); ggml_metal_cv_set_int16(cv, 1, FC_MUL_MV + 3); ggml_metal_cv_set_int16(cv, 1, FC_MUL_MV + 4); + ggml_metal_cv_set_bool (cv, split, FC_MUL_MV + 5); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -1321,11 +1483,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge(gg return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht(ggml_metal_library_t lib, int n) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht(ggml_metal_library_t lib, int n, ggml_type tsrc) { char base[256]; char name[256]; - snprintf(base, 256, "kernel_fwht_f32_%d", n); + snprintf(base, 256, "kernel_fwht_%s_%d", ggml_type_name(tsrc), n); snprintf(name, 256, "%s", base); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); @@ -1407,6 +1569,49 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge(ggml return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe( + ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm) { + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_topk_moe_f32"); + snprintf(name, 256, "%s_n_expert=%d_top_k=%d_with_norm=%d", base, n_expert, top_k, with_norm ? 1 : 0); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_bool (cv, with_norm, FC_TOPK_MOE + 0); + ggml_metal_cv_set_int32(cv, n_expert, FC_TOPK_MOE + 1); + ggml_metal_cv_set_int32(cv, top_k, FC_TOPK_MOE + 2); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_moe_reduce(ggml_metal_library_t lib, int32_t n_expert_used) { + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_moe_reduce_f32"); + snprintf(name, 256, "%s_n_expert_used=%d", base, n_expert_used); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_int32(cv, n_expert_used, FC_MOE_REDUCE + 0); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_pad( ggml_metal_library_t lib, const struct ggml_tensor * op, @@ -1861,7 +2066,48 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm(ggml_metal_ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { - res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_bool(cv, false, FC_NORM + 0); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } + + res.smem = 32*sizeof(float); + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm_scale(ggml_metal_library_t lib, const ggml_tensor * op) { + assert(op->op == GGML_OP_NORM || op->op == GGML_OP_RMS_NORM); + + GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); + + char base[256]; + char name[256]; + + const char * suffix = ""; + if (op->ne[0] % 4 == 0) { + suffix = "_4"; + } + + switch (op->op) { + case GGML_OP_NORM: snprintf(base, 256, "kernel_norm_mul_f32%s", suffix); break; + case GGML_OP_RMS_NORM: snprintf(base, 256, "kernel_rms_norm_mul_f32%s", suffix); break; + default: GGML_ABORT("fatal error"); + } + + snprintf(name, 256, "%s_use_scale", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_bool(cv, true, FC_NORM + 0); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); } res.smem = 32*sizeof(float); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 31fc07d44d47..1bdaecc737e2 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -126,9 +126,9 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_ad struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer (ggml_metal_library_t lib, const struct ggml_tensor * op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc (ggml_metal_library_t lib, enum ggml_op op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv (ggml_metal_library_t lib, const struct ggml_tensor * op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t nc, bool use_silu); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs, int32_t nc, bool use_silu); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tail); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan_ssd_mma (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rwkv (ggml_metal_library_t lib, const struct ggml_tensor * op); @@ -138,20 +138,26 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ex struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0 (ggml_metal_library_t lib, int ne02, int ne20); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax(ggml_metal_library_t lib); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax_part(ggml_metal_library_t lib); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge (ggml_metal_library_t lib, const struct ggml_tensor * op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht (ggml_metal_library_t lib, int n); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht (ggml_metal_library_t lib, int n, enum ggml_type tsrc); + struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_radix (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge (ggml_metal_library_t lib, const struct ggml_tensor * op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse ); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe (ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_moe_reduce (ggml_metal_library_t lib, int32_t n_expert_used); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin_one (ggml_metal_library_t lib, enum ggml_op op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_l2_norm (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_group_norm (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm_scale (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_im2col (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_1d (ggml_metal_library_t lib, const struct ggml_tensor * op); @@ -325,6 +331,11 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev); +struct ggml_metal_fusion_info; + +// the device-owned fusion debugging context (NULL unless fusion debugging is enabled) +struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev); + // // device buffers // diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index e20a4e89160b..81ea7f9d7a97 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1,4 +1,5 @@ #import "ggml-metal-device.h" +#import "ggml-metal-fusion.h" #import "ggml-impl.h" #import "ggml-backend-impl.h" @@ -896,6 +897,9 @@ void ggml_metal_encoder_end_encoding(ggml_metal_encoder_t encoder) { struct ggml_metal_device_props props; + // shared fusion debugging context + struct ggml_metal_fusion_info * finfo; + // virtual address for GPU memory allocations atomic_uintptr_t addr_virt; }; @@ -1249,7 +1253,7 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) { #endif dev->props.use_shared_buffers = dev->props.has_unified_memory; -#if TARGET_OS_OSX +#if TARGET_OS_OSX && TARGET_CPU_X86_64 // In case of eGPU, shared memory may be preferable. dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal; #endif @@ -1274,6 +1278,13 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) { dev->props.max_working_set_size = dev->mtl_device.maxBufferLength; } + { + const char * val = getenv("GGML_METAL_FUSION_DEBUG"); + dev->finfo = ggml_metal_fusion_info_init( + getenv("GGML_METAL_FUSION_DISABLE") == nil, + val ? atoi(val) : 0); + } + snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device); const char * gpu_name = [[dev->mtl_device name] UTF8String]; if (n_devices > 1) { @@ -1309,12 +1320,14 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) { } } +#if TARGET_CPU_X86_64 for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) { if ([dev->mtl_device supportsFamily:i]) { GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i); break; } } +#endif for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) { if ([dev->mtl_device supportsFamily:i]) { @@ -1348,6 +1361,8 @@ void ggml_metal_device_free(ggml_metal_device_t dev) { assert(dev != NULL); @autoreleasepool { + ggml_metal_fusion_info_free(dev->finfo); + ggml_metal_rsets_free(dev->rsets); ggml_metal_library_free(dev->library); @@ -1486,7 +1501,9 @@ static bool ggml_metal_supports_mul_mat_op( const struct ggml_tensor * op, bool src0_f16_has_mv, bool mm_path) { - if (!has_simdgroup_reduction || op->src[0]->type == GGML_TYPE_NVFP4) { + if (!has_simdgroup_reduction || + op->src[0]->type == GGML_TYPE_NVFP4 || + op->src[0]->type == GGML_TYPE_TQ1_0) { return false; } @@ -1718,6 +1735,12 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te op->src[0]->ne[0] != 576) { return false; } + if (op->src[1]->ne[0] == 72 && op->src[1]->ne[0] != op->src[2]->ne[0]) { + return false; + } + if (op->src[1]->ne[0] < op->src[2]->ne[0]) { + return false; + } if (op->src[1]->type != op->src[2]->type) { return false; } @@ -1786,8 +1809,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && - op->src[0]->ne[1] == 4 && - op->src[1]->ne[0] == 4 && ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]); case GGML_OP_DSV4_HC_POST: @@ -1795,16 +1816,15 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->src[2]->type == GGML_TYPE_F32 && - op->src[3]->type == GGML_TYPE_F32 && + (op->src[3] == NULL || op->src[3]->type == GGML_TYPE_F32) && op->type == GGML_TYPE_F32 && op->src[1]->ne[1] == 4 && op->src[2]->ne[0] == 4 && - op->src[3]->ne[0] == 4 && - op->src[3]->ne[1] == 4 && + (op->src[3] == NULL || (op->src[3]->ne[0] == 4 && op->src[3]->ne[1] == 4)) && ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && ggml_is_contiguous_rows(op->src[2]) && - ggml_is_contiguous_rows(op->src[3]); + (op->src[3] == NULL || ggml_is_contiguous_rows(op->src[3])); case GGML_OP_SSM_SCAN: return has_simdgroup_reduction; case GGML_OP_SSM_CONV: @@ -1817,6 +1837,12 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_SOLVE_TRI: return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_MUL_MAT: + // the FWHT kernels read an F16 source directly; every other F16 src1 path + // still goes through ggml_metal_supports_mul_mat_op + if (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F16 && + ggml_metal_op_mul_mat_use_fwht(op)) { + return has_simdgroup_reduction; + } return ggml_metal_supports_mul_mat_op( has_simdgroup_reduction, op, true, ggml_metal_op_mul_mat_use_mm(op, has_simdgroup_mm)); @@ -1887,7 +1913,8 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te }; } case GGML_OP_GET_ROWS: - return op->src[0]->type != GGML_TYPE_NVFP4; + return op->src[0]->type != GGML_TYPE_NVFP4 && + op->src[0]->type != GGML_TYPE_TQ1_0; case GGML_OP_SET_ROWS: { if (op->src[0]->type == GGML_TYPE_F16) { @@ -1932,6 +1959,10 @@ static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) { dev->props.has_tensor = false; } +struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev) { + return dev->finfo; +} + // // device buffers // diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp new file mode 100644 index 000000000000..eac3bd6fef5a --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp @@ -0,0 +1,1119 @@ +#include "ggml-metal-fusion.h" + +#include "ggml-backend-impl.h" +#include "ggml-metal-device.h" + +#include <algorithm> +#include <cstddef> +#include <cstring> +#include <set> +#include <string> +#include <vector> + +// derive the non-empty op sequence from the raw `ops_all` sequence +static std::vector<ggml_op> ggml_metal_fusion_filter_ops(const std::vector<ggml_op> & ops_all) { + std::vector<ggml_op> ops; + + for (ggml_op op : ops_all) { + if (!ggml_op_is_empty(op)) { + ops.push_back(op); + } + } + + return ops; +} + +struct ggml_metal_fusion { + ggml_metal_fusion_id id; + + std::vector<ggml_op> ops; // non-empty op sequence, derived from ops_all + std::vector<ggml_op> ops_all; // full raw op sequence (may include empty RESHAPE/VIEW nodes) + std::vector<int> outs; // additional fused output nodes, relative to ops + + // if unsafe: the generic chain/shape + ggml_can_fuse_subgraph checks are skipped and the + // check callback below is the sole validator (used for patterns that are not elision chains, + // e.g. the gdn + cache-cpy write-through fusion) + bool unsafe; + + // extra backend constraints on top of ggml_can_fuse_subgraph + // nodes[j] is the j-th node of the pattern; node_idxs[idx + j] is its raw graph index + bool (*check)(const struct ggml_metal_fusion * fusion, + const struct ggml_tensor * const * nodes, + const struct ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode); + + ggml_metal_fusion( + ggml_metal_fusion_id id, + const std::vector<ggml_op> & ops_all, + const std::vector<int> & outs, + bool unsafe, + bool (*check)(const struct ggml_metal_fusion * fusion, + const struct ggml_tensor * const * nodes, + const struct ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode)) + : id(id), + ops(ggml_metal_fusion_filter_ops(ops_all)), + ops_all(ops_all), + outs(outs), + unsafe(unsafe), + check(check) { + } +}; + +ggml_metal_fusion_id ggml_metal_fusion_get_id(const ggml_metal_fusion * fusion) { + return fusion->id; +} + +// ---- helpers ------------------------------------------------------------- + +// true if two tensors live in the same Metal buffer +static bool ggml_metal_fusion_same_buffer(const ggml_tensor * a, const ggml_tensor * b) { + if (!a || !b) { + return false; + } + + ggml_backend_buffer_t ba = a->view_src ? a->view_src->buffer : a->buffer; + ggml_backend_buffer_t bb = b->view_src ? b->view_src->buffer : b->buffer; + + ggml_metal_buffer_t ca = (ggml_metal_buffer_t) ba->context; + ggml_metal_buffer_t cb = (ggml_metal_buffer_t) bb->context; + + return ggml_metal_buffer_get_id(ca, a).metal == ggml_metal_buffer_get_id(cb, b).metal; +} + +// ---- pattern checks ------------------------------------------------------ + +// NORM/RMS_NORM + MUL + ADD: the weight/bias of each fused step must match the norm input +// width, be contiguous rows, and the fused outputs must stay F32 +static bool ggml_metal_fusion_check_norm( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(mode); + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + + GGML_ASSERT(fusion->ops.size() >= 2); + + if (fusion->id == GGML_METAL_FUSION_NORM_SCALE) { + GGML_ASSERT(fusion->ops.size() == 2); + + const ggml_tensor * scale = nodes[1]; + if (scale->op != GGML_OP_SCALE || scale->src[0] != nodes[0] || scale->src[1] || + scale->type != GGML_TYPE_F32) { + return false; + } + + return true; + } + + for (int j = 1; j < (int) fusion->ops.size(); j++) { + // the fused MUL/ADD must read the previous node as src0 + if (nodes[j]->src[0] != nodes[j - 1]) { + return false; + } + + // the weight/bias must have the same row width as the norm input + if (nodes[j]->src[1]->ne[0] != nodes[0]->ne[0]) { + return false; + } + + if (!ggml_is_contiguous_rows(nodes[j]->src[1])) { + return false; + } + + if (nodes[j]->type != GGML_TYPE_F32) { + return false; + } + } + + return true; +} + +// SSM_CONV + UNARY (silu) +static bool ggml_metal_fusion_check_ssm_conv_silu( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + GGML_UNUSED(mode); + + const ggml_tensor * conv = nodes[0]; + const ggml_tensor * un = nodes[1]; + + if (conv->op != GGML_OP_SSM_CONV || un->op != GGML_OP_UNARY || un->src[0] != conv || un->src[1]) { + return false; + } + + if (ggml_get_unary_op(un) != GGML_UNARY_OP_SILU) { + return false; + } + + if (conv->type != GGML_TYPE_F32 || un->type != GGML_TYPE_F32 || !ggml_is_contiguous_rows(un)) { + return false; + } + + return true; +} + +// ADD x N: each ADD reads the previous ADD as src0, and all addends must share layout +// (and, in FULL mode, live in the same Metal buffer) +static bool ggml_metal_fusion_check_add_chain( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + GGML_ASSERT(fusion->ops.size() >= 2); + + for (int j = 1; j < (int) fusion->ops.size(); j++) { + if (nodes[j]->src[0] != nodes[j - 1]) { + return false; + } + + if (!ggml_are_same_layout(nodes[j]->src[1], nodes[j - 1]->src[1])) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!ggml_metal_fusion_same_buffer(nodes[j]->src[1], nodes[0]->src[1])) { + return false; + } + } + } + + return true; +} + +// GATED_DELTA_NET + CPY: the trailing cpy scatters the gdn state snapshots into the recurrent +// cache, so the gdn kernel writes them straight to the cache and the cpy is elided. +// mirrors ggml_metal_op_can_fuse_gdn_cache (PR #25788). the gdn output has other consumers (the +// attn scores view), so unlike the other patterns this is not an elision chain: the structural +// checks live entirely in this callback (unsafe = true). +static bool ggml_metal_fusion_check_gdn_cache( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + + const ggml_tensor * gdn = nodes[0]; + const ggml_tensor * cpy = nodes[1]; + + // the kernel skips the snapshot tail, so the gdn output must not be a graph output + if (gdn->type != GGML_TYPE_F32 || (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + + if (cpy->op != GGML_OP_CPY || (cpy->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + + const int64_t S_v = gdn->src[2]->ne[0]; + const int64_t H = gdn->src[2]->ne[1]; + const int64_t n_tokens = gdn->src[2]->ne[2]; + const int64_t n_seqs = gdn->src[2]->ne[3]; + const int64_t K = ggml_get_op_params_i32(gdn, 0); + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs); + + const int64_t D = S_v * S_v * H; + const int64_t n_written = std::min<int64_t>(n_tokens, K); + + const ggml_tensor * src = cpy->src[0]; // gdn snapshot tail view + const ggml_tensor * dst = cpy->src[1]; // cache view + + // src must be this gdn's snapshot tail (contiguous, at the tail offset) + if (src->op != GGML_OP_VIEW || src->view_src != gdn || + src->view_offs != tail_off || !ggml_is_contiguous(src)) { + return false; + } + + const int64_t expected_ne[GGML_MAX_DIMS] = { D, n_seqs, n_written, 1 }; + if (dst->type != GGML_TYPE_F32 || + !std::equal(expected_ne, expected_ne + GGML_MAX_DIMS, dst->ne) || + dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || + dst->nb[1] != ggml_row_size(GGML_TYPE_F32, D)) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + // the cache must be allocated so the kernel can write straight to its buffer + if (dst->data == nullptr) { + return false; + } + } + + return true; +} + +// MUL + SIN + SQR + MUL + ADD (snake activation) +static bool ggml_metal_fusion_check_snake( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + GGML_UNUSED(mode); + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + + const ggml_tensor * mul0 = nodes[0]; + const ggml_tensor * sin_node = nodes[1]; + const ggml_tensor * sqr = nodes[2]; + const ggml_tensor * mul1 = nodes[3]; + const ggml_tensor * add = nodes[4]; + + // x carries the full activation shape, a is the broadcast operand + const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; + const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; + + // mul1 reads sqr and inv_b in either operand order + const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; + + // closure check: the trailing add reads the same x as the leading mul + const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0]; + + // x is in the supported whitelist and every chain intermediate shares x's type. + // a and inv_b bind as device const float * in the kernel, so they stay F32. + const bool types_ok = + (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) && + (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) && + (mul0->type == x->type) && (sin_node->type == x->type) && + (sqr->type == x->type) && (mul1->type == x->type) && + (add->type == x->type); + + // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D + const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; + const bool dim_ok = + (x->ne[2] == 1) && (x->ne[3] == 1) && + (add->ne[2] == 1) && (add->ne[3] == 1) && + (a->ne[2] == 1) && (a->ne[3] == 1) && + (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1); + + // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous + const bool contig_ok = + ggml_is_contiguous(x) && ggml_is_contiguous(add) && + ggml_is_contiguous(a) && ggml_is_contiguous(inv_b); + + return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x; +} + +#define GGML_METAL_TOPK_MOE_MAX_EXPERTS 1024 + +// SOFT_MAX + ARGSORT + GET_ROWS (plus optional norm/scale) for MoE routing. +// This is a multi-output elision chain: the fused kernel writes both the selected +// expert ids and the gathered/normalized routing weights. +static const std::vector<ggml_op> ops_topk_moe = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS +}; +static const std::vector<ggml_op> ops_topk_moe_scale = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_SCALE +}; +static const std::vector<ggml_op> ops_topk_moe_norm = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector<ggml_op> ops_topk_moe_norm_scale = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; + +static bool ggml_metal_fusion_check_topk_moe( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_ASSERT(fusion->ops.size() >= 3); + GGML_UNUSED(nodes); + + const int n_ops = (int) fusion->ops.size(); + + const bool with_norm = n_ops >= 6; + const bool with_scale = n_ops == 4 || n_ops == 7; + + // the fusion table operates on the non-empty node sequence; the raw graph also + // contains the RESHAPE/VIEW nodes that the fused kernel elides. + const std::vector<ggml_op> & ops_all = fusion->ops_all; + + const int raw_start = node_idxs[idx]; + int raw_end = node_idxs[idx + n_ops - 1]; + + // the norm variant ends with a RESHAPE that the non-empty sequence filters out; + // include it so the output use-count check sees the real final routing tensor + if (with_norm && !with_scale) { + if (raw_end + 1 >= gf->n_nodes) { + return false; + } + const ggml_tensor * trailing_reshape = gf->nodes[raw_end + 1]; + if (trailing_reshape->op != GGML_OP_RESHAPE || trailing_reshape->src[0] != gf->nodes[raw_end]) { + return false; + } + raw_end++; + } + + const int raw_count = raw_end - raw_start + 1; + if (raw_count != (int) ops_all.size()) { + return false; + } + + int raw_idxs[GGML_METAL_FUSION_MAX]; + for (int i = 0; i < raw_count; ++i) { + raw_idxs[i] = raw_start + i; + if (gf->nodes[raw_start + i]->op != ops_all[i]) { + return false; + } + } + + const ggml_tensor * softmax = gf->nodes[raw_start]; + const ggml_tensor * probs_reshaped = gf->nodes[raw_start + 1]; + const ggml_tensor * argsort = gf->nodes[raw_start + 2]; + const ggml_tensor * ids = gf->nodes[raw_start + 3]; + const ggml_tensor * get_rows = gf->nodes[raw_start + 4]; + const ggml_tensor * out = gf->nodes[raw_end]; + const ggml_tensor * logits = softmax->src[0]; + + // the fused kernel implements plain softmax only + float scale = 1.0f; + float max_bias = 0.0f; + memcpy(&scale, ((const int32_t *) softmax->op_params) + 0, sizeof(scale)); + memcpy(&max_bias, ((const int32_t *) softmax->op_params) + 1, sizeof(max_bias)); + if (scale != 1.0f || max_bias != 0.0f || softmax->src[1] || softmax->src[2]) { + return false; + } + + if (logits->type != GGML_TYPE_F32 || softmax->type != GGML_TYPE_F32 || + out->type != GGML_TYPE_F32 || ids->type != GGML_TYPE_I32) { + return false; + } + + const int64_t n_expert = logits->ne[0]; + const int64_t n_tokens = logits->ne[1]; + const int64_t n_expert_used = ids->ne[0]; + + if (n_expert <= 0 || n_tokens <= 0 || n_expert_used <= 0 || n_expert_used > n_expert || + n_expert > GGML_METAL_TOPK_MOE_MAX_EXPERTS || n_expert_used > GGML_METAL_TOPK_MOE_MAX_EXPERTS) { + return false; + } + + if (logits->ne[2] != 1 || logits->ne[3] != 1 || + ids->ne[1] != n_tokens || ids->ne[2] != 1 || ids->ne[3] != 1 || + out->ne[0] != 1 || out->ne[1] != n_expert_used || out->ne[2] != n_tokens || out->ne[3] != 1) { + return false; + } + + if (!ggml_is_contiguous(logits) || !ggml_is_contiguous(out) || + ids->nb[0] != ggml_type_size(GGML_TYPE_I32) || + ids->nb[1] != ggml_type_size(GGML_TYPE_I32) * n_expert) { + return false; + } + + if (probs_reshaped->src[0] != softmax || argsort->src[0] != softmax || + ids->src[0] != argsort || get_rows->src[0] != probs_reshaped || get_rows->src[1] != ids) { + return false; + } + + if (with_norm) { + const ggml_tensor * weights_reshaped = gf->nodes[raw_start + 5]; + const ggml_tensor * sum_rows = gf->nodes[raw_start + 6]; + const ggml_tensor * clamp = gf->nodes[raw_start + 7]; + const ggml_tensor * div = gf->nodes[raw_start + 8]; + const ggml_tensor * out_reshaped = gf->nodes[raw_start + 9]; + + if (weights_reshaped->src[0] != get_rows || sum_rows->src[0] != weights_reshaped || + clamp->src[0] != sum_rows || div->src[0] != weights_reshaped || div->src[1] != clamp || + out_reshaped->src[0] != div) { + return false; + } + + if (with_scale) { + const ggml_tensor * scale_node = gf->nodes[raw_start + 10]; + if (scale_node->src[0] != out_reshaped) { + return false; + } + } + } else if (with_scale) { + const ggml_tensor * scale_node = gf->nodes[raw_start + 5]; + if (scale_node->src[0] != get_rows) { + return false; + } + } + + const int outputs[2] = { raw_start + 3, raw_end }; + if (!ggml_can_fuse_subgraph_ext(gf, raw_idxs, raw_count, ops_all.data(), outputs, 2)) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!logits->data || !out->data || !ids->data) { + return false; + } + } + + return true; +} + +#define GGML_METAL_MOE_REDUCE_MAX_EXPERTS 8 + +struct ggml_metal_moe_reduce_match { + const ggml_tensor * experts; + const ggml_tensor * weights; + const ggml_tensor * dst; + int node_count; +}; + +static bool ggml_metal_fusion_match_moe_reduce( + const ggml_cgraph * gf, int node_idx, const std::vector<ggml_op> & ops_all, + ggml_metal_moe_reduce_match * match) { + if (match == nullptr || node_idx < 0 || node_idx + (int) ops_all.size() > gf->n_nodes) { + return false; + } + + const ggml_tensor * mul = gf->nodes[node_idx]; + if (mul->op != GGML_OP_MUL || mul->type != GGML_TYPE_F32) { + return false; + } + + // MUL, then one VIEW per expert, then one ADD per additional expert + const int raw_count = (int) ops_all.size(); + const int n_expert_used = raw_count / 2; + + if (n_expert_used < 2 || n_expert_used > GGML_METAL_MOE_REDUCE_MAX_EXPERTS || + raw_count != 2 * n_expert_used) { + return false; + } + + int n_views = 0; + while (node_idx + 1 + n_views < gf->n_nodes && + gf->nodes[node_idx + 1 + n_views]->op == GGML_OP_VIEW) { + n_views++; + } + + if (n_views != n_expert_used) { + return false; + } + + for (int i = n_expert_used + 1; i < raw_count; ++i) { + if (gf->nodes[node_idx + i]->op != GGML_OP_ADD) { + return false; + } + } + + int raw_idxs[GGML_METAL_FUSION_MAX]; + for (int i = 0; i < raw_count; ++i) { + raw_idxs[i] = node_idx + i; + if (gf->nodes[node_idx + i]->op != ops_all[i]) { + return false; + } + } + + const ggml_tensor * experts = mul->src[0]; + const ggml_tensor * weights = mul->src[1]; + const ggml_tensor * dst = gf->nodes[node_idx + raw_count - 1]; + + if (experts->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + const int64_t n_embd = experts->ne[0]; + const int64_t n_tokens = experts->ne[2]; + + if (n_embd <= 0 || n_tokens <= 0 || experts->ne[1] != n_expert_used || experts->ne[3] != 1 || + weights->ne[0] != 1 || weights->ne[1] != n_expert_used || weights->ne[2] != n_tokens || weights->ne[3] != 1 || + dst->ne[0] != n_embd || dst->ne[1] != n_tokens || dst->ne[2] != 1 || dst->ne[3] != 1) { + return false; + } + + if (!ggml_is_contiguous(experts) || !ggml_is_contiguous(weights) || !ggml_is_contiguous(dst)) { + return false; + } + + for (int i = 1; i <= n_expert_used; ++i) { + const ggml_tensor * view = gf->nodes[node_idx + i]; + if (view->view_src != mul || view->src[0] != mul || + view->view_offs != (size_t) (i - 1) * mul->nb[1] || + view->ne[0] != n_embd || view->ne[1] != n_tokens || + view->nb[1] != mul->nb[2]) { + return false; + } + } + + const ggml_tensor * prev_add = nullptr; + for (int j = 1; j < n_expert_used; ++j) { + const ggml_tensor * add = gf->nodes[node_idx + n_expert_used + j]; + const ggml_tensor * rhs = gf->nodes[node_idx + j + 1]; + const ggml_tensor * lhs = j == 1 ? gf->nodes[node_idx + 1] : prev_add; + if (add->src[0] != lhs || add->src[1] != rhs) { + return false; + } + prev_add = add; + } + + const int outputs[1] = { node_idx + raw_count - 1 }; + if (!ggml_can_fuse_subgraph_ext(gf, raw_idxs, raw_count, ops_all.data(), outputs, 1)) { + return false; + } + + match->experts = experts; + match->weights = weights; + match->dst = dst; + match->node_count = raw_count; + return true; +} + +static bool ggml_metal_fusion_check_moe_reduce( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(nodes); + + ggml_metal_moe_reduce_match match; + if (!ggml_metal_fusion_match_moe_reduce(gf, node_idxs[idx], fusion->ops_all, &match)) { + return false; + } + + if ((int) fusion->ops.size() != match.experts->ne[1]) { + return false; + } + + const int raw_end = node_idxs[idx] + match.node_count - 1; + if (node_idxs[idx + (int) fusion->ops.size() - 1] != raw_end) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!match.experts->data || !match.weights->data || !match.dst->data) { + return false; + } + } + + return true; +} + +// ---- patterns ------------------------------------------------------------ + +static const std::vector<ggml_op> ops_norm_mul = { GGML_OP_NORM, GGML_OP_MUL }; +static const std::vector<ggml_op> ops_norm_mul_add = { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD }; +static const std::vector<ggml_op> ops_norm_scale = { GGML_OP_NORM, GGML_OP_SCALE }; +static const std::vector<ggml_op> ops_rms_norm_mul = { GGML_OP_RMS_NORM, GGML_OP_MUL }; +static const std::vector<ggml_op> ops_rms_norm_mul_add = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }; +static const std::vector<ggml_op> ops_rms_norm_scale = { GGML_OP_RMS_NORM, GGML_OP_SCALE }; + +static const std::vector<ggml_op> ops_add_2 = { GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector<ggml_op> ops_add_3 = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector<ggml_op> ops_add_4 = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector<ggml_op> ops_add_5 = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector<ggml_op> ops_add_6 = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector<ggml_op> ops_add_7 = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector<ggml_op> ops_snake = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD }; + +static const std::vector<ggml_op> ops_gdn_cache = { GGML_OP_GATED_DELTA_NET, GGML_OP_CPY }; + +static const std::vector<ggml_op> ops_ssm_conv_silu = { GGML_OP_SSM_CONV, GGML_OP_UNARY }; + +static const std::vector<ggml_op> ops_moe_reduce_2 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_ADD +}; +static const std::vector<ggml_op> ops_moe_reduce_3 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_ADD, GGML_OP_ADD +}; +static const std::vector<ggml_op> ops_moe_reduce_4 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, + GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD +}; +static const std::vector<ggml_op> ops_moe_reduce_5 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, + GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD +}; +static const std::vector<ggml_op> ops_moe_reduce_6 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, + GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD +}; +static const std::vector<ggml_op> ops_moe_reduce_7 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, + GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD +}; +static const std::vector<ggml_op> ops_moe_reduce_8 = { + GGML_OP_MUL, + GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, + GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD +}; + +static const std::vector<ggml_metal_fusion> ggml_metal_fusions = { + { GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_SCALE, ops_norm_scale, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL, ops_rms_norm_mul, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_rms_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_SCALE, ops_rms_norm_scale, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_2, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_3, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_4, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_5, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_6, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_7, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_SNAKE, ops_snake, {}, false, ggml_metal_fusion_check_snake }, + { GGML_METAL_FUSION_GDN_CACHE, ops_gdn_cache, {}, true, ggml_metal_fusion_check_gdn_cache }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_2, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_3, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_4, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_5, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_6, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_7, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_8, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_SSM_CONV_SILU, ops_ssm_conv_silu, {}, false, ggml_metal_fusion_check_ssm_conv_silu }, +}; + +// ---- alloc deps ----------------------------------------------------------- + +static bool ggml_metal_fusion_match_raw_pattern( + const ggml_cgraph * gf, int node_idx, const std::vector<ggml_op> & ops) { + if (node_idx < 0 || node_idx + (int) ops.size() > gf->n_nodes) { + return false; + } + + for (int i = 0; i < (int) ops.size(); ++i) { + if (gf->nodes[node_idx + i]->op != ops[i]) { + return false; + } + } + + return true; +} + +static void ggml_metal_fusion_add_pattern_alloc_deps( + void * user_data, + void (*add_alloc_dep)(void *, ggml_tensor *, ggml_tensor *), + ggml_cgraph * gf, + const ggml_metal_fusion * fusion, + int node_idx) { + const int last_node = node_idx + (int) fusion->ops_all.size() - 1; + + // keep all external inputs alive until the fused output + std::set<ggml_tensor *> seen; + for (int j = 0; j < (int) fusion->ops_all.size(); ++j) { + ggml_tensor * node = gf->nodes[node_idx + j]; + for (int s = 0; s < GGML_MAX_SRC; ++s) { + ggml_tensor * src = node->src[s]; + if (src && seen.insert(src).second) { + add_alloc_dep(user_data, src, gf->nodes[last_node]); + } + } + seen.insert(node); + } +} + +void ggml_metal_fusion_add_alloc_deps( + void * user_data, + void (*add_alloc_dep)(void *, ggml_tensor *, ggml_tensor *), + ggml_cgraph * gf) { + for (int i = 0; i < gf->n_nodes; ++i) { + const ggml_metal_fusion * best = nullptr; + int best_raw = 0; + + for (const ggml_metal_fusion & fusion : ggml_metal_fusions) { + if ((int) fusion.ops_all.size() <= best_raw) { + continue; + } + if (ggml_metal_fusion_match_raw_pattern(gf, i, fusion.ops_all)) { + best = &fusion; + best_raw = (int) fusion.ops_all.size(); + } + } + + if (best) { + ggml_metal_fusion_add_pattern_alloc_deps(user_data, add_alloc_dep, gf, best, i); + i += best_raw - 1; + } + } +} + +// ---- shared fusion info --------------------------------------------------- + +static std::string ggml_metal_fusion_label(const ggml_metal_fusion * fusion) { + GGML_ASSERT(fusion != nullptr); + + std::string label; + for (int j = 0; j < (int) fusion->ops.size(); j++) { + if (j > 0) { + label += '+'; + } + label += ggml_op_name(fusion->ops[j]); + } + return label; +} + +struct ggml_metal_fusion_info { + std::vector<std::string> labels; + std::vector<uint64_t> counts; + bool enabled; + bool stats; + bool labels_set; + int debug; +}; + +struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug) { + ggml_metal_fusion_info * finfo = new ggml_metal_fusion_info; + finfo->enabled = enabled; + finfo->stats = debug > 0; + finfo->labels_set = false; + finfo->debug = debug; + + if (finfo->stats) { + ggml_metal_fusion_info_labels_init(finfo); + } + + return finfo; +} + +void ggml_metal_fusion_info_free(ggml_metal_fusion_info * finfo) { + delete finfo; +} + +bool ggml_metal_fusion_info_enabled(const ggml_metal_fusion_info * finfo) { + return finfo->enabled; +} + +bool ggml_metal_fusion_info_stats(const ggml_metal_fusion_info * finfo) { + return finfo->stats; +} + +int ggml_metal_fusion_info_debug(const ggml_metal_fusion_info * finfo) { + return finfo->debug; +} + +int ggml_metal_fusion_info_n_fusions(const ggml_metal_fusion_info * finfo) { + return (int) finfo->labels.size(); +} + +const char * ggml_metal_fusion_info_label(const ggml_metal_fusion_info * finfo, int idx) { + GGML_ASSERT(idx >= 0 && idx < (int) finfo->labels.size()); + return finfo->labels[idx].c_str(); +} + +uint64_t ggml_metal_fusion_info_count(const ggml_metal_fusion_info * finfo, int idx) { + GGML_ASSERT(idx >= 0 && idx < (int) finfo->counts.size()); + return finfo->counts[idx]; +} + +void ggml_metal_fusion_info_count_fusion(ggml_metal_fusion_info * finfo, const ggml_metal_fusion * fusion) { + if (!finfo->stats || fusion == nullptr) { + return; + } + + const ptrdiff_t idx = fusion - ggml_metal_fusions.data(); + if (idx >= 0 && idx < (ptrdiff_t) finfo->counts.size()) { + finfo->counts[idx]++; + } +} + +void ggml_metal_fusion_info_set_enabled(ggml_metal_fusion_info * finfo, bool enabled) { + finfo->enabled = enabled; +} + +void ggml_metal_fusion_info_labels_init(ggml_metal_fusion_info * finfo) { + if (finfo->labels_set) { + return; + } + + finfo->labels.clear(); + finfo->counts.assign(ggml_metal_fusions.size(), 0); + finfo->labels.reserve(ggml_metal_fusions.size()); + + for (const ggml_metal_fusion & fusion : ggml_metal_fusions) { + finfo->labels.emplace_back(ggml_metal_fusion_label(&fusion)); + } + + finfo->labels_set = true; +} + +void ggml_metal_fusion_info_stats_init(ggml_metal_fusion_info * finfo) { + finfo->stats = true; + ggml_metal_fusion_info_labels_init(finfo); +} + +void ggml_metal_fusion_info_stats_reset(ggml_metal_fusion_info * finfo) { + std::fill(finfo->counts.begin(), finfo->counts.end(), 0); +} + +int ggml_metal_fusion_info_stats_get(const ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n) { + const int n_fusions = (int) finfo->labels.size(); + + if (labels == nullptr) { + return n_fusions; + } + + const int n_fill = std::min(n, n_fusions); + for (int i = 0; i < n_fill; i++) { + labels[i] = finfo->labels[i].c_str(); + if (counts != nullptr) { + counts[i] = finfo->counts[i]; + } + } + + return n_fill; +} + +// ---- memory-range checks ------------------------------------------------- + +// reject fusions where an external source overlaps any fused output. the fused +// kernels elide intermediate nodes, so only sources that are not part of the +// fused subgraph can cause read/write races with the output. +static bool ggml_metal_fusion_check_memory_ranges( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + int node_count) { + // some fused kernels write through a tensor that also appears as a source (e.g. the gdn + // cache cpy), so a source that is the same memory as the output is not an external read + // source + auto same_memory = [](const ggml_tensor * a, const ggml_tensor * b) { + if (a->data && b->data && a->data == b->data) { + return true; + } + for (const ggml_tensor * v = a; v; v = v->view_src) { + if (v == b) { + return true; + } + } + for (const ggml_tensor * v = b; v; v = v->view_src) { + if (v == a) { + return true; + } + } + return false; + }; + + auto nodes_overlap = [](const ggml_tensor * a, const ggml_tensor * b) { + if (!a || !b || !a->data || !b->data || !a->buffer || !b->buffer) { + return false; + } + + if (a->buffer != b->buffer) { + return false; + } + + const int64_t a_start = (int64_t) a->data; + const int64_t a_end = a_start + ggml_backend_buft_get_alloc_size(a->buffer->buft, a); + const int64_t b_start = (int64_t) b->data; + const int64_t b_end = b_start + ggml_backend_buft_get_alloc_size(b->buffer->buft, b); + + return (b_start <= a_start && a_start < b_end) || + (a_start <= b_start && b_start < a_end); + }; + + auto is_intermediate = [](const ggml_tensor * src, const ggml_tensor * const * nodes, int j) { + for (int k = 0; k < j; ++k) { + if (src == nodes[k]) { + return true; + } + for (const ggml_tensor * view_src = src->view_src; view_src; view_src = view_src->view_src) { + if (view_src == nodes[k]) { + return true; + } + } + } + return false; + }; + + auto check_dst = [&](const ggml_tensor * dst) { + for (int j = 0; j < node_count; ++j) { + for (int s = 0; s < GGML_MAX_SRC; ++s) { + const ggml_tensor * src = nodes[j]->src[s]; + if (!src || src->op == GGML_OP_NONE || same_memory(src, dst)) { + continue; + } + + if (nodes_overlap(dst, src) && !is_intermediate(src, nodes, j)) { + return false; + } + } + } + return true; + }; + + if (!check_dst(nodes[node_count - 1])) { + return false; + } + + for (int offset : fusion->outs) { + GGML_ASSERT(offset >= 0 && offset < node_count); + if (!check_dst(nodes[offset])) { + return false; + } + } + + return true; +} + +// ---- queries ------------------------------------------------------------- + +// find the longest pattern matching the node sequence starting at idx +// (idx is a position in node_idxs, which maps to graph node indices) +const ggml_metal_fusion * ggml_metal_fusion_next( + const ggml_cgraph * gf, + const int * node_idxs, + int n_idxs, + int idx, + ggml_metal_fusion_mode mode, + int * n_out) { + const ggml_metal_fusion * res = nullptr; + int best = 1; + + for (const ggml_metal_fusion & fusion : ggml_metal_fusions) { + const int n_ops = (int) fusion.ops.size(); + + // only look for a longer match than the current best + if (n_ops <= best) { + continue; + } + if (idx + n_ops > n_idxs) { + continue; + } + + const ggml_tensor * nodes[GGML_METAL_FUSION_MAX]; + + // the op sequence must match exactly + bool ok = true; + for (int j = 0; j < n_ops; j++) { + nodes[j] = gf->nodes[node_idxs[idx + j]]; + if (nodes[j]->op != fusion.ops[j]) { + ok = false; + break; + } + } + if (!ok) { + continue; + } + + if (!fusion.unsafe) { + // common element-wise chain constraints: each node reads the previous one, + // and all nodes have the same shape + for (int j = 1; j < n_ops && ok; j++) { + if (nodes[j]->src[0] != nodes[j - 1] && nodes[j]->src[1] != nodes[j - 1]) { + ok = false; + break; + } + if (!ggml_are_same_shape(nodes[j], nodes[j - 1])) { + ok = false; + break; + } + } + if (!ok) { + continue; + } + + // primary output is the last node; additional outputs come from fusion.outs + int outputs_buf[GGML_METAL_FUSION_MAX]; + outputs_buf[0] = node_idxs[idx + n_ops - 1]; + for (size_t i = 0; i < fusion.outs.size(); ++i) { + const int out_offset = fusion.outs[i]; + GGML_ASSERT(out_offset >= 0 && out_offset < n_ops); + outputs_buf[i + 1] = node_idxs[idx + out_offset]; + } + + const int n_outputs = 1 + (int) fusion.outs.size(); + + // structural subgraph checks (op sequence, elidable uses, view containment) + if (!ggml_can_fuse_subgraph_ext(gf, node_idxs + idx, n_ops, fusion.ops.data(), outputs_buf, n_outputs)) { + continue; + } + } + + // pattern-specific checks (the sole validator for unsafe patterns) + if (fusion.check && !fusion.check(&fusion, nodes, gf, node_idxs, idx, mode)) { + continue; + } + + // the compute phase has allocated tensors and can detect aliasing between + // external sources and fused outputs; the optimizer phase cannot do this yet + if (mode == GGML_METAL_FUSION_FULL && + !ggml_metal_fusion_check_memory_ranges(&fusion, nodes, n_ops)) { + continue; + } + + best = n_ops; + res = &fusion; + } + + *n_out = best; + + return res; +} + +// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that +// could be fused, chaining patterns back-to-back. matching runs on the same filtered (view +// transparent) node sequence that the compute phase uses, so the returned count is the raw index +// span from idx to the last matched node (intermediate views are packed along). +int ggml_metal_fusion_max(const ggml_cgraph * gf, int idx) { + // an empty/view node cannot start a pattern - pack it alone + if (ggml_op_is_empty(gf->nodes[idx]->op) || ggml_is_empty(gf->nodes[idx])) { + return 1; + } + + // collect the non-empty node indices starting at idx + int idxs[GGML_METAL_FUSION_MAX]; + int n_idxs = 0; + for (int i = idx; i < gf->n_nodes && n_idxs < GGML_METAL_FUSION_MAX; i++) { + if (!ggml_op_is_empty(gf->nodes[i]->op) && !ggml_is_empty(gf->nodes[i])) { + idxs[n_idxs++] = i; + } + } + if (n_idxs == 0) { + return 1; + } + + int total = 0; + int i_f = 0; + + while (i_f < n_idxs && total < GGML_METAL_FUSION_MAX) { + int len = 1; + const ggml_metal_fusion * fusion = ggml_metal_fusion_next(gf, idxs, n_idxs, i_f, GGML_METAL_FUSION_STRUCTURAL, &len); + if (!fusion || total + len > GGML_METAL_FUSION_MAX) { + break; + } + + total += len; + i_f += len; + } + + if (i_f == 0) { + return 1; + } + + // map the matched non-empty nodes back to the raw index span (views are included) + return std::min(GGML_METAL_FUSION_MAX, idxs[i_f - 1] - idx + 1); +} diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-metal-fusion.h new file mode 100644 index 000000000000..6b139a69b597 --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-fusion.h @@ -0,0 +1,98 @@ +// single source of truth for the fusions supported by the Metal backend +// +// every fusable subgraph is declared exactly once as a ggml_metal_fusion entry in +// the table in ggml-metal-fusion.cpp. both the graph optimizer (ggml_metal_fusion_max) +// and the op encoders (ggml_metal_fusion_next) consult this same table, so the two +// phases can never disagree about what can be fused. + +#pragma once + +#include "ggml-impl.h" + +#include <stdbool.h> + +#ifdef __cplusplus +extern "C" { +#endif + +// the maximum number of nodes that can be fused in a single kernel +// (also the maximum length of a packed fusion group during graph optimization) +#define GGML_METAL_FUSION_MAX 16 + +typedef enum ggml_metal_fusion_mode { + // structural checks only; used by the graph optimizer, at which point the graph + // tensors are not allocated yet, so buffer placement cannot be verified + GGML_METAL_FUSION_STRUCTURAL = 0, + // full checks, including buffer placement; used by the op encoders + GGML_METAL_FUSION_FULL, +} ggml_metal_fusion_mode; + +// identifier of each fusion pattern so the op encoders know which kernel to use +typedef enum ggml_metal_fusion_id { + GGML_METAL_FUSION_NONE = 0, + GGML_METAL_FUSION_NORM_MUL, // NORM/RMS_NORM + MUL + GGML_METAL_FUSION_NORM_MUL_ADD, // NORM/RMS_NORM + MUL + ADD + GGML_METAL_FUSION_NORM_SCALE, // NORM/RMS_NORM + SCALE + GGML_METAL_FUSION_ADD_CHAIN, // ADD x N (N in [2, 7]) + GGML_METAL_FUSION_SNAKE, // MUL + SIN + SQR + MUL + ADD + GGML_METAL_FUSION_GDN_CACHE, // GATED_DELTA_NET + CPY (write snapshots into the recurrent cache) + GGML_METAL_FUSION_TOPK_MOE, // SOFT_MAX + ARGSORT + GET_ROWS + norm/scale (MoE routing) + GGML_METAL_FUSION_MOE_REDUCE, // MUL + expert VIEWs + ADD chain (MoE output reduction) + GGML_METAL_FUSION_SSM_CONV_SILU, // SSM_CONV + UNARY (silu) +} ggml_metal_fusion_id; + +struct ggml_metal_fusion; // defined in ggml-metal-fusion.cpp + +typedef struct ggml_metal_fusion ggml_metal_fusion; + +// access the fusion identifier without exposing the full pattern definition +ggml_metal_fusion_id ggml_metal_fusion_get_id(const struct ggml_metal_fusion * fusion); + +// apply any alloc-dependencies required by the fused kernels during graph optimize +void ggml_metal_fusion_add_alloc_deps( + void * user_data, + void (*add_alloc_dep)(void *, struct ggml_tensor *, struct ggml_tensor *), + struct ggml_cgraph * gf); + +// ---- shared fusion info --------------------------------------------------- + +// shared fusion debugging context, owned by the device; newly created backend contexts for that +// device register with it so the fusion counters are race-free and accumulate across contexts. +struct ggml_metal_fusion_info; // defined in ggml-metal-fusion.cpp + +struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug); +void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo); + +bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo); +bool ggml_metal_fusion_info_stats (const struct ggml_metal_fusion_info * finfo); +int ggml_metal_fusion_info_debug (const struct ggml_metal_fusion_info * finfo); + +int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo); +const char * ggml_metal_fusion_info_label (const struct ggml_metal_fusion_info * finfo, int idx); +uint64_t ggml_metal_fusion_info_count (const struct ggml_metal_fusion_info * finfo, int idx); + +void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion); +void ggml_metal_fusion_info_set_enabled (struct ggml_metal_fusion_info * finfo, bool enabled); + +void ggml_metal_fusion_info_stats_init ( struct ggml_metal_fusion_info * finfo); +void ggml_metal_fusion_info_stats_reset( struct ggml_metal_fusion_info * finfo); +int ggml_metal_fusion_info_stats_get (const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n); +void ggml_metal_fusion_info_labels_init( struct ggml_metal_fusion_info * finfo); + +// compute phase: longest fusion starting at idx (a position in node_idxs) that matches in `mode`. +// returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes consumed. +const ggml_metal_fusion * ggml_metal_fusion_next( + const struct ggml_cgraph * gf, + const int * node_idxs, + int n_idxs, + int idx, + ggml_metal_fusion_mode mode, + int * n_out); + +// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that +// could be fused, chaining patterns back-to-back. returns at least 1. +int ggml_metal_fusion_max(const struct ggml_cgraph * gf, int idx); + +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 30e40f527f90..490dd83a1569 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -14,6 +14,8 @@ #define N_MM_SIMD_GROUP_X 2 #define N_MM_SIMD_GROUP_Y 2 +#define N_MM_NPART_AMAX 256 + // kernel parameters for mat-vec threadgroups // // N_R0: number of src0 rows to process per simdgroup @@ -62,24 +64,31 @@ #define N_R0_IQ1_S 4 #define N_SG_IQ1_S 2 +#define N_R0_IQ1_S_SPLIT 8 #define N_R0_IQ1_M 4 #define N_SG_IQ1_M 2 +#define N_R0_IQ1_M_SPLIT 8 #define N_R0_IQ2_XXS 4 #define N_SG_IQ2_XXS 2 +#define N_R0_IQ2_XXS_SPLIT 8 #define N_R0_IQ2_XS 4 #define N_SG_IQ2_XS 2 +#define N_R0_IQ2_XS_SPLIT 8 #define N_R0_IQ2_S 4 #define N_SG_IQ2_S 2 +#define N_R0_IQ2_S_SPLIT 8 #define N_R0_IQ3_XXS 4 #define N_SG_IQ3_XXS 2 +#define N_R0_IQ3_XXS_SPLIT 8 #define N_R0_IQ3_S 4 #define N_SG_IQ3_S 2 +#define N_R0_IQ3_S_SPLIT 8 #define N_R0_IQ4_NL 2 #define N_SG_IQ4_NL 2 @@ -107,6 +116,10 @@ #define FC_SUM_ROWS 1400 #define FC_UPSCALE 1500 #define FC_GATED_DELTA_NET 1600 +#define FC_NORM 1700 +#define FC_TOPK_MOE 1800 +#define FC_MOE_REDUCE 1900 +#define FC_DSV4_HC 2000 // op-specific constants #define OP_FLASH_ATTN_EXT_NQPSG 8 @@ -548,6 +561,14 @@ typedef struct { uint64_t nb21; } ggml_metal_kargs_mul_mm_id_map0; +typedef struct { + int32_t ne00; + int32_t ne01; + int32_t ne02; + uint64_t nb01; + uint64_t nb02; +} ggml_metal_kargs_mul_mm_id_amax; + typedef struct { int32_t ne00; int32_t ne02; @@ -605,6 +626,7 @@ typedef struct { uint64_t nbf1[3]; uint64_t nbf2[3]; uint64_t nbf3[3]; + float scale; } ggml_metal_kargs_norm; typedef struct { @@ -893,7 +915,6 @@ typedef struct { uint64_t nb00; uint64_t nb01; uint64_t nb02; - int64_t ne10; int64_t ne11; uint64_t nb10; uint64_t nb11; @@ -978,6 +999,7 @@ typedef struct { uint64_t nb1; uint64_t nb2; uint64_t nb3; + uint64_t nb_out; // 0 => snapshots are appended after the attn scores (unfused) } ggml_metal_kargs_gated_delta_net; typedef struct { @@ -1213,6 +1235,19 @@ typedef struct { int32_t top_k; // k } ggml_metal_kargs_top_k; +typedef struct { + int32_t ne01; // n_tokens + uint64_t nb01; // logits row stride + uint64_t nb1_ids; // ids row stride + float clamp; + float scale; +} ggml_metal_kargs_topk_moe; + +typedef struct { + int32_t ne00; // n_embd + int32_t ne02; // n_tokens +} ggml_metal_kargs_moe_reduce; + typedef struct { int32_t nrows; } ggml_metal_kargs_fwht; @@ -1265,8 +1300,10 @@ typedef struct { uint64_t nb_x2; uint64_t nb_w0; uint64_t nb_w1; + uint64_t nb_w2; uint64_t nb_d0; uint64_t nb_d1; + float scale; } ggml_metal_kargs_dsv4_hc_pre; typedef struct { diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 3db8bca43752..708703e51192 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -7,6 +7,7 @@ #include "ggml-metal-impl.h" #include "ggml-metal-common.h" #include "ggml-metal-device.h" +#include "ggml-metal-fusion.h" #include "ggml-metal-tuning.h" #include <cassert> @@ -31,24 +32,22 @@ struct ggml_metal_op { ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, ggml_cgraph * gf, + ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion) { + int debug_graph) { this->dev = dev; this->lib = ggml_metal_device_get_library(dev); this->enc = ggml_metal_encoder_init(cmd_buf, use_concurrency); this->mem_ranges = ggml_mem_ranges_init(debug_graph); + this->finfo = finfo; this->idx_start = idx_start; this->idx_end = idx_end; - this->use_fusion = use_fusion; this->use_concurrency = use_concurrency; this->use_capture = use_capture; this->debug_graph = debug_graph; - this->debug_fusion = debug_fusion; this->gf = gf; idxs.reserve(gf->n_nodes); @@ -78,15 +77,24 @@ struct ggml_metal_op { return ggml_graph_node(gf, idxs[i]); } - bool can_fuse(int i0, const ggml_op * ops, int n_ops) const { - assert(use_fusion); + // consult the fusion table for the longest pattern starting at i0 + // returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes + const ggml_metal_fusion * can_fuse(int i0, enum ggml_metal_fusion_mode mode, int * n_out) const { + assert(use_fusion()); assert(i0 >= 0 && i0 < n_nodes()); - if (i0 + n_ops > n_nodes()) { - return false; - } + return ggml_metal_fusion_next(gf, idxs.data(), (int) idxs.size(), i0, mode, n_out); + } - return ggml_can_fuse_ext(gf, idxs.data() + i0, ops, n_ops); + // whether to attempt fusion; the toggle lives in the shared fusion debugging context owned + // by the device (initialized from GGML_METAL_FUSION_DISABLE, overridable by the test) + bool use_fusion() const { + return ggml_metal_fusion_info_enabled(finfo); + } + + // record that a fusion fired, indexed by the matching table entry + void count_fusions(const ggml_metal_fusion * fusion) const { + ggml_metal_fusion_info_count_fusion(finfo, fusion); } ggml_metal_device_t dev; @@ -94,12 +102,13 @@ struct ggml_metal_op { ggml_metal_encoder_t enc; ggml_mem_ranges_t mem_ranges; - bool use_fusion; + // shared fusion debugging context + ggml_metal_fusion_info * finfo; + bool use_concurrency; bool use_capture; int debug_graph; - int debug_fusion; private: ggml_cgraph * gf; @@ -115,24 +124,22 @@ ggml_metal_op_t ggml_metal_op_init( ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, ggml_cgraph * gf, + ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion) { + int debug_graph) { ggml_metal_op_t res = new ggml_metal_op( dev, cmd_buf, gf, + finfo, idx_start, idx_end, - use_fusion, use_concurrency, use_capture, - debug_graph, - debug_fusion); + debug_graph); return res; } @@ -219,7 +226,16 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { // otherwise, we add the new ranges to the encoding context and process the node concurrently // { - const bool is_concurrent = ggml_metal_op_concurrency_check(ctx, node); + bool is_concurrent = ggml_metal_op_concurrency_check(ctx, node); + + if (is_concurrent && ctx->use_fusion()) { + int n_fuse = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n_fuse); + if (fusion) { + // fused kernels write to the last node of the group, not necessarily to the first node's dst + is_concurrent = ggml_mem_ranges_check(ctx->mem_ranges, ctx->node(idx + n_fuse - 1)); + } + } if (!is_concurrent) { ggml_metal_op_concurrency_reset(ctx); @@ -1398,7 +1414,7 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); ggml_metal_encoder_t enc = ctx->enc; - auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op->op); + auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op); ggml_metal_encoder_set_pipeline(enc, pipeline); @@ -1450,7 +1466,6 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { GGML_ASSERT(x->type == GGML_TYPE_F32); GGML_ASSERT(weights->type == GGML_TYPE_F32); GGML_ASSERT(op->type == GGML_TYPE_F32); - GGML_ASSERT(x->ne[1] == 4); ggml_metal_kargs_dsv4_hc_pre args = { /*.n_embd =*/ (int32_t) x->ne[0], @@ -1460,8 +1475,10 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { /*.nb_x2 =*/ x->nb[2], /*.nb_w0 =*/ weights->nb[0], /*.nb_w1 =*/ weights->nb[1], + /*.nb_w2 =*/ weights->nb[2], /*.nb_d0 =*/ op->nb[0], /*.nb_d1 =*/ op->nb[1], + /*.scale =*/ ggml_get_op_params_f32(op, 0), }; ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); @@ -1484,7 +1501,6 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { GGML_ASSERT(x->type == GGML_TYPE_F32); GGML_ASSERT(residual->type == GGML_TYPE_F32); GGML_ASSERT(post->type == GGML_TYPE_F32); - GGML_ASSERT(comb->type == GGML_TYPE_F32); GGML_ASSERT(op->type == GGML_TYPE_F32); GGML_ASSERT(residual->ne[1] == 4); @@ -1498,9 +1514,9 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { /*.nb_r2 =*/ residual->nb[2], /*.nb_p0 =*/ post->nb[0], /*.nb_p1 =*/ post->nb[1], - /*.nb_c0 =*/ comb->nb[0], - /*.nb_c1 =*/ comb->nb[1], - /*.nb_c2 =*/ comb->nb[2], + /*.nb_c0 =*/ comb ? comb->nb[0] : 0, + /*.nb_c1 =*/ comb ? comb->nb[1] : 0, + /*.nb_c2 =*/ comb ? comb->nb[2] : 0, /*.nb_d0 =*/ op->nb[0], /*.nb_d1 =*/ op->nb[1], /*.nb_d2 =*/ op->nb[2], @@ -1510,8 +1526,12 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(x), 1); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(residual), 2); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(post), 3); - ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(comb), 4); - ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5); + if (comb) { + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(comb), 4); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5); + } else { + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 4); + } const int n_tiles = (args.n_embd + 31)/32; const int nsg = std::min(4, n_tiles); @@ -1528,6 +1548,14 @@ int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { int ggml_metal_op_soft_max(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); + if (ctx->use_fusion()) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_TOPK_MOE) { + return ggml_metal_op_topk_moe(ctx, idx); + } + } + ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; @@ -1628,6 +1656,20 @@ int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + int n_fuse = 1; + bool use_silu = false; + + if (ctx->use_fusion()) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_SSM_CONV_SILU) { + n_fuse = n; + use_silu = true; + + ctx->count_fusions(fusion); + } + } + ggml_metal_kargs_ssm_conv args = { /*.ne00 =*/ ne00, /*.ne01 =*/ ne01, @@ -1635,7 +1677,6 @@ int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { /*.nb00 =*/ nb00, /*.nb01 =*/ nb01, /*.nb02 =*/ nb02, - /*.ne10 =*/ ne10, /*.ne11 =*/ ne11, /*.nb10 =*/ nb10, /*.nb11 =*/ nb11, @@ -1647,6 +1688,8 @@ int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { /*.nb2 =*/ nb2, }; + const ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(n_fuse > 1 ? ctx->node(idx + n_fuse - 1) : op); + // Use batched kernel for prefill (ne1 > 1) to reduce threadgroup dispatch overhead const bool use_batched = (ne1 > 1); @@ -1661,31 +1704,35 @@ int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { else if (ne1 > 4 ) BATCH_SIZE = 8; else BATCH_SIZE = 2; - auto pipeline = ggml_metal_library_get_pipeline_ssm_conv_batched(lib, op, BATCH_SIZE); + auto pipeline = ggml_metal_library_get_pipeline_ssm_conv_batched(lib, op, BATCH_SIZE, (int32_t) ne10, use_silu); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); - ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); + ggml_metal_encoder_set_buffer(enc, bid_dst, 3); // Dispatch: ne01 rows, ceil(ne1/BATCH_SIZE) token batches, ne02 sequences // Each threadgroup has BATCH_SIZE threads, each handling one token const int n_token_batches = (ne1 + BATCH_SIZE - 1) / BATCH_SIZE; ggml_metal_encoder_dispatch_threadgroups(enc, ne01, n_token_batches, ne02, BATCH_SIZE, 1, 1); } else { - auto pipeline = ggml_metal_library_get_pipeline_ssm_conv(lib, op); + auto pipeline = ggml_metal_library_get_pipeline_ssm_conv(lib, op, (int32_t) ne10, use_silu); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); - ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); + ggml_metal_encoder_set_buffer(enc, bid_dst, 3); ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne1, ne02, 1, 1, 1); } - return 1; + if (n_fuse > 1 && ggml_metal_fusion_info_debug(ctx->finfo) > 1) { + GGML_LOG_DEBUG("%s: fuse: SSM_CONV + UNARY\n", __func__); + } + + return n_fuse; } int ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) { @@ -1868,6 +1915,8 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; + const bool use_fusion = ctx->use_fusion(); + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -1880,6 +1929,31 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { auto pipeline = ggml_metal_library_get_pipeline_gated_delta_net(lib, op); + // when fused with the trailing cache cpy, the snapshots are written straight into the + // recurrent cache and the cpy is skipped (see GGML_METAL_FUSION_GDN_CACHE) + ggml_metal_buffer_id bid_out = ggml_metal_get_buffer_id(op); + uint64_t nb_out = 0; + int n_fuse = 1; + + if (use_fusion) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_GDN_CACHE) { + const ggml_tensor * dst_cache = ctx->node(idx + 1)->src[1]; // cache view + + bid_out = ggml_metal_get_buffer_id(dst_cache); + nb_out = dst_cache->nb[2]/sizeof(float); + n_fuse = 2; + + ctx->count_fusions(fusion); + + if (debug_fusion > 1) { + GGML_LOG_DEBUG("%s: fuse: GATED_DELTA_NET + CPY\n", __func__); + } + } + } + int ida = 0; ggml_metal_kargs_gated_delta_net args = { @@ -1918,23 +1992,25 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { /*.nb1 =*/ nb1, /*.nb2 =*/ nb2, /*.nb3 =*/ nb3, + /*.nb_out =*/ nb_out, }; ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); // args ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); // q ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); // k ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++); // v ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++); // gate ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); // beta ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); // state - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst (attn) + ggml_metal_encoder_set_buffer (enc, bid_out, ida++); // state_out const int nsg = pipeline.nsg; ggml_metal_encoder_dispatch_threadgroups(enc, op->src[2]->ne[0]/nsg, op->src[2]->ne[1], op->src[2]->ne[3], 32, nsg, 1); - return 1; + return n_fuse; } int ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) { @@ -2242,12 +2318,6 @@ int ggml_metal_op_pool_1d(ggml_metal_op_t ctx, int idx) { return 1; } -// supported FWHT sizes, must stay in sync with the -// kernel_fwht_f32_<N> templates in ggml-metal.metal -static bool ggml_metal_fwht_supported_size(int64_t n) { - return n == 64 || n == 128 || n == 256 || n == 512; -} - int ggml_metal_op_fwht(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -2263,7 +2333,7 @@ int ggml_metal_op_fwht(ggml_metal_op_t ctx, int idx) { /*.nrows = */ (int32_t) nrows, }; - auto pipeline = ggml_metal_library_get_pipeline_fwht(lib, n); + auto pipeline = ggml_metal_library_get_pipeline_fwht(lib, n, src1->type); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); @@ -2349,17 +2419,8 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const int32_t hint = ggml_get_op_params_i32(op, 1); - - if (hint == GGML_HINT_SRC0_IS_HADAMARD) { - if (op->src[1]->type == GGML_TYPE_F32 && - op->type == GGML_TYPE_F32 && - ggml_is_contiguous(op->src[1]) && - ggml_is_contiguous(op) && - ggml_are_same_shape(op->src[1], op) && - ggml_metal_fwht_supported_size(op->src[1]->ne[0])) { - return ggml_metal_op_fwht(ctx, idx); - } + if (ggml_metal_op_mul_mat_use_fwht(op)) { + return ggml_metal_op_fwht(ctx, idx); } const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev); @@ -2595,6 +2656,15 @@ size_t ggml_metal_op_mul_mat_id_extra_ids(const ggml_tensor * op) { return ggml_type_size(GGML_TYPE_I32)*ne02*ne21; } +size_t ggml_metal_op_mul_mat_id_extra_amax(const ggml_tensor * op) { + assert(op->op == GGML_OP_MUL_MAT_ID); + + GGML_UNUSED(op); + + // 2 scaling factors (8 bytes) + N_MM_NPART_AMAX per-threadgroup scales for stage-1 + return 8 + N_MM_NPART_AMAX*sizeof(float); +} + int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -2646,6 +2716,39 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { ggml_metal_buffer_id bid_ids = bid_tpe; bid_ids.offs += ggml_metal_op_mul_mat_id_extra_tpe(op); + ggml_metal_buffer_id bid_amax = bid_ids; + bid_amax.offs += ggml_metal_op_mul_mat_id_extra_ids(op); + + // src1 prec [TAG_GGML_PREC] + const bool use_amax = ggml_get_op_params_i32(op, 3) == GGML_PREC_F32; + + // src1 rescale factors, computed before the matmul + // ref: https://github.com/ggml-org/llama.cpp/pull/26223 + if (use_amax) { + ggml_metal_kargs_mul_mm_id_amax args = { + /*.ne00 =*/ ne10, + /*.ne01 =*/ ne11, + /*.ne02 =*/ ne12, + /*.nb01 =*/ nb11, + /*.nb02 =*/ nb12, + }; + + auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_amax_part(lib); + + const size_t smem = pipeline.smem; + + GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, bid_src1, 1); + ggml_metal_encoder_set_buffer (enc, bid_amax, 2); + + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + + ggml_metal_encoder_dispatch_threadgroups(enc, N_MM_NPART_AMAX, 1, 1, 256, 1, 1); + } + { ggml_metal_kargs_mul_mm_id_map0 args = { ne02, @@ -2677,9 +2780,20 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, ne02, 1, 1); } - // this barrier is always needed because the next kernel has to wait for the id maps to be computed ggml_metal_op_concurrency_reset(ctx); + if (use_amax) { + auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_amax(lib); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_buffer (enc, bid_amax, 0); + + ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, 32, 1, 1); + + // the next kernel has to wait for the amax data + ggml_metal_op_concurrency_reset(ctx); + } + { auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id(lib, op); @@ -2709,6 +2823,7 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer (enc, bid_tpe, 3); ggml_metal_encoder_set_buffer (enc, bid_ids, 4); ggml_metal_encoder_set_buffer (enc, bid_dst, 5); + ggml_metal_encoder_set_buffer (enc, bid_amax, 6); const size_t smem = pipeline.smem; @@ -2887,6 +3002,7 @@ static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op) const bool dk_dv_ok = (dk == 32 && dv == 32) || (dk == 64 && dv == 64) || (dk == 96 && dv == 96) || + (dk == 96 && dv == 64) || (dk == 128 && dv == 128) || (dk == 192 && dv == 128) || (dk == 192 && dv == 192) || @@ -3468,7 +3584,6 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { auto cfg = use_sparse ? ggml_metal_tuning::fa_vec_baseline_cfg((int) ne00, (int) ne20) : ggml_metal_tuning::fa_vec_pick( - props_dev->device_id, props_dev->gpu_family, (int) op->src[1]->type, (int) ne00, (int) ne20, // dk, dv (ne00 == dk for FA) @@ -3718,56 +3833,26 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { return 1; } -// Snake activation autofuse: mul -> sin -> sqr -> mul -> add -static bool ggml_metal_op_can_fuse_snake(ggml_metal_op_t ctx, int idx) { - static constexpr ggml_op snake_ops[5] = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD }; - - if (ctx->node(idx)->op != GGML_OP_MUL || !ctx->can_fuse(idx, snake_ops, 5)) { - return false; - } - - const ggml_tensor * mul0 = ctx->node(idx + 0); - const ggml_tensor * sin_node = ctx->node(idx + 1); - const ggml_tensor * sqr = ctx->node(idx + 2); - const ggml_tensor * mul1 = ctx->node(idx + 3); - const ggml_tensor * add = ctx->node(idx + 4); +int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { + int n_fuse = 1; + const ggml_metal_fusion * fusion = nullptr; - // x carries the full activation shape, a is the broadcast operand - const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; - const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; + if (ctx->use_fusion()) { + int n = 1; + fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + n_fuse = n; - // mul1 reads sqr and inv_b in either operand order - const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; - - // closure check: the trailing add reads the same x as the leading mul - const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0]; - - // x is in the supported whitelist and every chain intermediate shares x's type. - // a and inv_b bind as device const float * in the kernel, so they stay F32. - const bool types_ok = - (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) && - (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) && - (mul0->type == x->type) && (sin_node->type == x->type) && - (sqr->type == x->type) && (mul1->type == x->type) && - (add->type == x->type); - // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D - const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; - const bool dim_ok = - (x->ne[2] == 1) && (x->ne[3] == 1) && - (add->ne[2] == 1) && (add->ne[3] == 1) && - (a->ne[2] == 1) && (a->ne[3] == 1) && - (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1); - // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous - const bool contig_ok = - ggml_is_contiguous(x) && ggml_is_contiguous(add) && - ggml_is_contiguous(a) && ggml_is_contiguous(inv_b); - - return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x; -} + // snake activation autofuse: mul -> sin -> sqr -> mul -> add + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_SNAKE) { + ctx->count_fusions(fusion); + return ggml_metal_op_snake_fused(ctx, idx); + } -int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { - if (ctx->use_fusion && ggml_metal_op_can_fuse_snake(ctx, idx)) { - return ggml_metal_op_snake_fused(ctx, idx); + // MoE output reduction: experts * weights -> weighted sum + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_MOE_REDUCE) { + ctx->count_fusions(fusion); + return ggml_metal_op_moe_reduce(ctx, idx); + } } ggml_tensor * op = ctx->node(idx); @@ -3775,9 +3860,9 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; + const bool use_fusion = ctx->use_fusion(); - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -3822,57 +3907,19 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { /*.o1 =*/ { bid_src1.offs }, }; - ggml_op fops[8]; - - int n_fuse = 1; - // c[0] = add(a, b[0]) // c[1] = add(c[0], b[1]) // c[2] = add(c[1], b[2]) // ... - if (use_fusion) { - fops[0] = GGML_OP_ADD; - fops[1] = GGML_OP_ADD; - fops[2] = GGML_OP_ADD; - fops[3] = GGML_OP_ADD; - fops[4] = GGML_OP_ADD; - fops[5] = GGML_OP_ADD; - fops[6] = GGML_OP_ADD; - fops[7] = GGML_OP_ADD; - - // note: in metal, we sometimes encode the graph in parallel so we have to avoid fusing ops - // across splits. idx_end indicates the last node in the current split - for (n_fuse = 0; n_fuse <= 6; ++n_fuse) { - if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) { - break; - } - - ggml_tensor * f0 = ctx->node(idx + n_fuse); - ggml_tensor * f1 = ctx->node(idx + n_fuse + 1); - - if (f0 != f1->src[0]) { - break; - } - - // b[0] === b[1] === ... - if (!ggml_are_same_layout(f0->src[1], f1->src[1])) { - break; - } - - // only fuse ops if src1 is in the same Metal buffer - ggml_metal_buffer_id bid_fuse = ggml_metal_get_buffer_id(f1->src[1]); - if (bid_fuse.metal != bid_src1.metal) { - break; - } - - //ctx->fuse_cnt[ops[n_fuse + 1]->op]++; - - args.o1[n_fuse + 1] = bid_fuse.offs; + if (use_fusion && fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_ADD_CHAIN) { + // the offsets of the fused addends are relative to the start of the src1 buffer + for (int i = 1; i < n_fuse; i++) { + args.o1[i] = ggml_metal_get_buffer_id(ctx->node(idx + i)->src[1]).offs; } - ++n_fuse; + ctx->count_fusions(fusion); - if (debug_fusion > 1 && n_fuse > 1) { + if (debug_fusion > 1) { GGML_LOG_DEBUG("%s: fuse: ADD x %d\n", __func__, n_fuse); } } @@ -4080,9 +4127,9 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; + const bool use_fusion = ctx->use_fusion(); - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -4108,67 +4155,61 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { /*.nbf1 =*/ { nb01 }, /*.nbf2 =*/ { nb02 }, /*.nbf3 =*/ { nb03 }, + /*.scale =*/ 1.0f, }; - ggml_op fops[8]; - int n_fuse = 1; + bool fused_norm_scale = false; ggml_metal_buffer_id bid_fuse[2] = { bid_src0, bid_src0 }; // d[0] = norm(a) - // d[1] = mul(d[0], b) + // d[1] = mul(d[0], b) or scale(d[0]) // d[2] = add(d[1], c) if (use_fusion) { - fops[0] = op->op; - fops[1] = GGML_OP_MUL; - fops[2] = GGML_OP_ADD; + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); - for (n_fuse = 0; n_fuse <= 1; ++n_fuse) { - if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) { - break; - } + if (fusion && (ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_NORM_MUL || ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_NORM_MUL_ADD)) { + n_fuse = n; - ggml_tensor * f0 = ctx->node(idx + n_fuse); - ggml_tensor * f1 = ctx->node(idx + n_fuse + 1); + ctx->count_fusions(fusion); - if (f0 != f1->src[0]) { - break; - } + for (int i = 1; i < n_fuse; i++) { + const ggml_tensor * fn = ctx->node(idx + i); - if (f1->src[1]->ne[0] != op->ne[0]) { - break; - } + bid_fuse[i - 1] = ggml_metal_get_buffer_id(fn->src[1]); - if (!ggml_is_contiguous_rows(f1->src[1])) { - break; - } + args.nef1[i] = fn->src[1]->ne[1]; + args.nef2[i] = fn->src[1]->ne[2]; + args.nef3[i] = fn->src[1]->ne[3]; - if (f1->type != GGML_TYPE_F32) { - break; + args.nbf1[i] = fn->src[1]->nb[1]; + args.nbf2[i] = fn->src[1]->nb[2]; + args.nbf3[i] = fn->src[1]->nb[3]; } - //ctx->fuse_cnt[f1->op]++; - - bid_fuse[n_fuse] = ggml_metal_get_buffer_id(f1->src[1]); + if (debug_fusion > 1) { + if (n_fuse == 2) { + GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op)); + } + if (n_fuse == 3) { + GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op)); + } + } + } - args.nef1[n_fuse + 1] = f1->src[1]->ne[1]; - args.nef2[n_fuse + 1] = f1->src[1]->ne[2]; - args.nef3[n_fuse + 1] = f1->src[1]->ne[3]; + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_NORM_SCALE) { + n_fuse = n; + fused_norm_scale = true; - args.nbf1[n_fuse + 1] = f1->src[1]->nb[1]; - args.nbf2[n_fuse + 1] = f1->src[1]->nb[2]; - args.nbf3[n_fuse + 1] = f1->src[1]->nb[3]; - } + ctx->count_fusions(fusion); - ++n_fuse; + const ggml_tensor * scale_node = ctx->node(idx + 1); + args.scale = ggml_get_op_params_f32(scale_node, 0); - if (debug_fusion > 1 && n_fuse > 1) { - if (n_fuse == 2) { - GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op)); - } - if (n_fuse == 3) { - GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op)); + if (debug_fusion > 1) { + GGML_LOG_DEBUG("%s: fuse: %s + SCALE\n", __func__, ggml_op_name(op->op)); } } } @@ -4185,7 +4226,9 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { } } - auto pipeline = ggml_metal_library_get_pipeline_norm(lib, op, n_fuse); + auto pipeline = fused_norm_scale ? + ggml_metal_library_get_pipeline_norm_scale(lib, op) : + ggml_metal_library_get_pipeline_norm(lib, op, n_fuse); int nth = 32; // SIMD width @@ -5406,6 +5449,110 @@ static void ggml_metal_op_top_k_radix(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); } +int ggml_metal_op_topk_moe(ggml_metal_op_t ctx, int idx) { + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + int n_fuse = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n_fuse); + if (!fusion || ggml_metal_fusion_get_id(fusion) != GGML_METAL_FUSION_TOPK_MOE) { + return 1; + } + + ggml_tensor * softmax = ctx->node(idx); + ggml_tensor * logits = softmax->src[0]; + ggml_tensor * get_rows = ctx->node(idx + 2); + ggml_tensor * ids = get_rows->src[1]; + ggml_tensor * weights = ctx->node(idx + n_fuse - 1); + + const int64_t n_expert = logits->ne[0]; + const int64_t n_tokens = logits->ne[1]; + const int64_t n_expert_used = ids->ne[0]; + + const bool with_norm = n_fuse >= 6; + const bool with_scale = n_fuse == 4 || n_fuse == 7; + + float clamp = -INFINITY; + if (with_norm) { + ggml_tensor * clamp_node = ctx->node(idx + 4); + clamp = ggml_get_op_params_f32(clamp_node, 0); + } + + float scale = 1.0f; + if (with_scale) { + ggml_tensor * scale_node = ctx->node(idx + n_fuse - 1); + scale = ggml_get_op_params_f32(scale_node, 0); + } + + ggml_metal_kargs_topk_moe args = { + /*.ne01 =*/ (int32_t) n_tokens, + /*.nb01 =*/ logits->nb[1], + /*.nb1_ids =*/ ids->nb[1], + /*.clamp =*/ clamp, + /*.scale =*/ scale, + }; + + auto pipeline = ggml_metal_library_get_pipeline_topk_moe(lib, (int32_t) n_expert, (int32_t) n_expert_used, with_norm); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(logits), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(weights), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(ids), 3); + + ggml_metal_encoder_dispatch_threadgroups(enc, (uint32_t) n_tokens, 1, 1, 32, 1, 1); + + ctx->count_fusions(fusion); + + if (ggml_metal_fusion_info_debug(ctx->finfo) > 1) { + GGML_LOG_DEBUG("%s: fuse: SOFT_MAX + ARGSORT + GET_ROWS\n", __func__); + } + + return n_fuse; +} + +int ggml_metal_op_moe_reduce(ggml_metal_op_t ctx, int idx) { + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + int n_fuse = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n_fuse); + if (!fusion || ggml_metal_fusion_get_id(fusion) != GGML_METAL_FUSION_MOE_REDUCE) { + return 1; + } + + ggml_tensor * mul = ctx->node(idx); + ggml_tensor * experts = mul->src[0]; + ggml_tensor * weights = mul->src[1]; + ggml_tensor * dst = ctx->node(idx + n_fuse - 1); + + ggml_metal_kargs_moe_reduce args = { + /*.ne00 =*/ (int32_t) experts->ne[0], + /*.ne02 =*/ (int32_t) experts->ne[2], + }; + + auto pipeline = ggml_metal_library_get_pipeline_moe_reduce(lib, (int32_t) experts->ne[1]); + + const int nth = std::min(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + const int n_col_tiles = (args.ne00 + nth - 1) / nth; + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(experts), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(weights), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(dst), 3); + + ggml_metal_encoder_dispatch_threadgroups(enc, (uint32_t) args.ne02, (uint32_t) n_col_tiles, 1, nth, 1, 1); + + ctx->count_fusions(fusion); + + if (ggml_metal_fusion_info_debug(ctx->finfo) > 1) { + GGML_LOG_DEBUG("%s: fuse: MOE_REDUCE\n", __func__); + } + + return n_fuse; +} + int ggml_metal_op_top_k(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index f8fe50b468e4..583d1156bf27 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -8,17 +8,18 @@ extern "C" { typedef struct ggml_metal_op * ggml_metal_op_t; +struct ggml_metal_fusion; // forward decl (ggml-metal-device.h) + ggml_metal_op_t ggml_metal_op_init( ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, struct ggml_cgraph * gf, + struct ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion); + int debug_graph); void ggml_metal_op_free(ggml_metal_op_t ctx); @@ -35,6 +36,7 @@ size_t ggml_metal_op_mul_mat_id_extra_tpe(const struct ggml_tensor * op); // id map [n_tokens, n_expert] size_t ggml_metal_op_mul_mat_id_extra_ids(const struct ggml_tensor * op); +size_t ggml_metal_op_mul_mat_id_extra_amax(const struct ggml_tensor * op); // return true if we should use the FA vector kernel for this op bool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op); @@ -96,6 +98,8 @@ int ggml_metal_op_timestep_embedding(ggml_metal_op_t ctx, int idx); int ggml_metal_op_argmax (ggml_metal_op_t ctx, int idx); int ggml_metal_op_argsort (ggml_metal_op_t ctx, int idx); int ggml_metal_op_top_k (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_topk_moe (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_moe_reduce (ggml_metal_op_t ctx, int idx); int ggml_metal_op_tri (ggml_metal_op_t ctx, int idx); int ggml_metal_op_opt_step_adamw (ggml_metal_op_t ctx, int idx); int ggml_metal_op_opt_step_sgd (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp index 8cdc55a0ac90..217ad43b311c 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.cpp +++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp @@ -34,6 +34,9 @@ int fa_vec_baseline_ne(int dk, int dv) { if (dk == 96 && dv == 96) { return 4; } + if (dk == 96 && dv == 64) { + return 4; + } if (dk == 128 && dv == 128) { return 1; } @@ -63,3775 +66,1053 @@ fa_vec_cfg_t fa_vec_baseline_cfg(int dk, int dv) { } // Generated by `ggml-metal-tuning fa-vec`; do not hand-edit. -// One row per kept bucket, plus per-(dtype,dk,dv) ne11-collapsed domain defaults -// (ne11_b = FA_VEC_NE11_DEFAULT, ne01_b = domain). To retune or add a device, re-run the -// sweep and paste its output. See ggml-metal-tuning.h for the row/lookup semantics. +// Keyed by Apple GPU family, retagged from the per-SKU token the tuner emits, and pooled from +// the sweeps listed at the head of each segment. One row per kept bucket, plus +// per-(dtype,dk,dv) ne11-collapsed domain defaults (ne11_b = FA_VEC_NE11_DEFAULT, +// ne01_b = domain). To retune or add a device, see tools/tuning/README.md. +// See ggml-metal-tuning.h for the row/lookup semantics. // ref: https://github.com/ggml-org/llama.cpp/pull/27824 +// https://github.com/ggml-org/llama.cpp/discussions/27668 constexpr fa_vec_entry_t fa_vec_tuned_table[] = { - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 128, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 128, 128, 1, 1 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 128, 128, 1, 3 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 320, 256, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 320, 256, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 96, 96, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 96, 96, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 96, 96, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 128, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 128, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 192, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 192, 192, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 192, 192, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 192, 128, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 192, 128, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 192, 128, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 512, 512, 3, 3 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 320, 256, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 512, 512, 2, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 512, 512, 3, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 128, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 128, 128, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, 3, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 320, 256, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 320, 256, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, 3, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 128, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 192, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 320, 256, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 320, 256, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 128, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 128, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 128, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 128, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 256, 256, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 256, 256, 3, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 320, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 2, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 256, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 128, 128, 1, 1 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 128, 128, 1, 3 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 320, 256, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 320, 256, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 512, 512, 2, 3 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 512, 512, 3, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 96, 96, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 192, 192, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 576, 512, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 576, 512, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 576, 512, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_1, 576, 512, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 128, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 192, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 320, 256, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 320, 256, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 320, 256, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 320, 256, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, 2, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 192, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 3, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 320, 256, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 320, 256, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 512, 512, 2, 0 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 512, 512, 2, 1 }, { 4, 1 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 512, 512, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 256, 256, 2, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 256, 256, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 512, 512, 3, 1 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 320, 256, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 64, 64, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 192, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 256, 256, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 256, 256, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 96, 96, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 96, 96, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, 2, 4 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, 3, 4 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 128, 128, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 128, 128, 2, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 192, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 192, 2, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 2, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 3, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 3, 2 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 3, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 1, 1 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 512, 512, 2, 0 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 512, 512, 3, 0 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 512, 512, 3, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 512, 512, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 576, 512, 2, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 576, 512, 2, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 576, 512, 3, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 64, 64, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, 3, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 192, 3, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 320, 256, 3, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 320, 256, 3, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 512, 512, 2, 0 }, { 4, 1 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 512, 512, 2, 4 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 512, 512, 3, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 512, 512, 3, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q4_1, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 128, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 320, 256, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 512, 512, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_0, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 128, 128, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 576, 512, 3, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q5_1, 576, 512, 3, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, 3, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 256, 256, 3, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 320, 256, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 128, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 256, 256, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 512, 512, 3, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 256, 256, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, 3, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, 3, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, 3, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, 3, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 192, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 128, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 256, 256, 2, 3 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 320, 256, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 2, 0 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 3, 0 }, { 4, 1 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 2, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 3, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 576, 512, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 576, 512, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 576, 512, 3, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 192, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 96, 96, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 64, 64, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } }, - - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 64, 64, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 96, 96, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 128, 128, 1, 1 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 192, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 512, 512, 3, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 512, 512, 3, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 320, 256, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 192, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 192, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 128, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 32, 32, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 96, 96, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 128, 128, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 256, 256, 2, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 256, 256, 2, 4 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 320, 256, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 128, 128, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 256, 256, 2, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 256, 256, 2, 4 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 512, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 512, 512, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 192, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 256, 256, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 192, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 256, 256, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 512, 512, 3, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 512, 512, 3, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, 2, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, 3, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 256, 256, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 4 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 320, 256, 3, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } }, - - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 320, 256, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 4 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 96, 96, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 4 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 96, 96, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 3, 2 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 3, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 256, 256, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 2, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 2, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 3, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 3, 2 }, { 4, 1 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 3, 4 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 576, 512, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 576, 512, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 128, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 512, 512, 3, 0 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 512, 512, 2, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 512, 512, 3, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 64, 64, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 64, 64, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 192, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 192, 128, 3, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 512, 512, 2, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 512, 512, 2, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 512, 512, 3, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 512, 512, 1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 512, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 512, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 3, 0 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 3, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 96, 96, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 1, 2 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 128, 1, 4 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 128, 3, 2 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 512, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 512, 512, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 512, 512, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 512, 512, 3, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 512, 512, 3, 4 }, { 4, 1 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 2, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, 3, 4 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, 3, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, 2, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 1, 4 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 2, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 3, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 320, 256, 2, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 320, 256, 3, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 2, 3 }, { 4, 1 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 3, 1 }, { 4, 1 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 3, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 3, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 3, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 3, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, 3, 0 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 3, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, 3, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, 2, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 2, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 2, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 3, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 576, 512, 2, 0 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 576, 512, 2, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 576, 512, 3, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 2 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, 1, 0 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, 2, 4 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, 3, 4 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 3, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 3, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 3, 0 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 3, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, 2, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, 3, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 2, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 2, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 3, 3 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 64, 64, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 4 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, 3, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 256, 256, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 2, 0 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 1, 3 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 2, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 2, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 3, 1 }, { 4, 1 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 3, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 576, 512, 2, 1 }, { 4, 4 } }, - - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 256, 256, 2, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 256, 256, 3, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 512, 512, 3, 0 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 512, 512, 3, 2 }, { 4, 1 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 512, 512, 3, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 576, 512, 2, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 576, 512, 3, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 320, 256, 3, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 512, 512, 2, 0 }, { 4, 1 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 512, 512, 3, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 512, 512, 3, 2 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 512, 512, 2, 3 }, { 2, 1 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 512, 512, 3, 1 }, { 2, 1 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 512, 512, 3, 3 }, { 2, 1 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 576, 512, 3, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 576, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 576, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 576, 512, 1, 4 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 512, 512, 3, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 576, 512, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 1, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 128, 128, 2, 3 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } }, + // Apple7 - M1, M1_ULTRA + { { 7, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 128, 128, 2, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 320, 256, 1, 1 }, { 1, 2 } }, + { { 7, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } }, + { { 7, GGML_TYPE_F16, 320, 256, 1, 3 }, { 1, 2 } }, + { { 7, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 1, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 1, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, 1, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 256, 256, 3, 2 }, { 1, 1 } }, + { { 7, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 32, 32, 1, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 96, 96, 1, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, 1, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, 1, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } }, + { { 7, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } }, + { { 7, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, 1, 2 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, 1, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, 1, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, 2, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, 3, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 192, 1, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 128, 2, 3 }, { 4, 2 } }, + { { 7, GGML_TYPE_Q5_1, 192, 128, 3, 3 }, { 4, 2 } }, + { { 7, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 256, 256, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, 2, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 1, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 3, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, 2, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, + { { 7, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + // Apple8 - M2, M2_MAX, M2_PRO + { { 8, GGML_TYPE_F16, 64, 64, 1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 2 } }, + { { 8, GGML_TYPE_F16, 64, 64, 3, 2 }, { 1, 2 } }, + { { 8, GGML_TYPE_F16, 64, 64, 3, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_F16, 64, 64, 3, 4 }, { 1, 2 } }, + { { 8, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 128, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 320, 256, 2, 3 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 320, 256, 2, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 96, 96, 1, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, 1, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 320, 256, 3, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 1, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 1, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 2, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 3, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 3, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 1, 3 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 1, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 576, 512, 1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 576, 512, 1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 576, 512, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 576, 512, 1, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, 1, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, 2, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, 2, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 2 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, 1, 4 }, { 1, 2 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, 1, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 32, 32, 1, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 1, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 3, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, + { { 8, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + // Apple9 - A18_PRO, M3_MAX, M3_PRO, M3_ULTRA, M4, M4_MAX, M4_PRO + { { 9, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 64, 64, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, 1, 3 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_F16, 128, 128, 1, 1 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_F16, 192, 192, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 128, 2, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 256, 256, 2, 0 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 256, 256, 3, 0 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 64, 64, 2, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_0, 64, 64, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 3, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 192, 1, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 192, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 128, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 320, 256, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 320, 256, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 96, 96, 3, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 1 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, 1, 4 }, { 1, 1 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 1 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, 1, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 320, 256, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 320, 256, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 320, 256, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 320, 256, 3, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 1, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 1, 4 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, 1, 2 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, 1, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 96, 96, 1, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 1, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 2, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, 1, 4 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 320, 256, 2, 3 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, 1, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, 1, 3 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, 1, 4 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, 1, 2 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, 1, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 96, 96, 1, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, 1, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, 3, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, 1, 4 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q5_1, 320, 256, 3, 3 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 96, 96, 3, 2 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 192, 1, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 192, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 192, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, 1, 3 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } }, + // Apple10 - M5_MAX + { { 10, GGML_TYPE_F16, 32, 32, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 32, 32, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 32, 32, 1, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 32, 32, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 32, 32, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 64, 64, 3, 0 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 64, 64, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 64, 64, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 64, 64, 3, 3 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 64, 64, 3, 4 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 96, 96, 3, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, 3, 0 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 128, 128, 1, 2 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, + { { 10, GGML_TYPE_F16, 128, 128, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 192, 128, 1, 4 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 192, 128, 3, 2 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 1 } }, + { { 10, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } }, + { { 10, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 512, 512, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 512, 512, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 512, 512, 2, 3 }, { 4, 2 } }, + { { 10, GGML_TYPE_F16, 512, 512, 3, 3 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 512, 512, 3, 4 }, { 4, 1 } }, + { { 10, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 576, 512, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 576, 512, 1, 1 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 576, 512, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 576, 512, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 576, 512, 2, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 576, 512, 2, 3 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, 3, 4 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 1, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, 1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, 2, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, 3, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, 2, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, 2, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 1, 4 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 2, 3 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 2, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 3, 3 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 3, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 320, 256, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 320, 256, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 320, 256, 2, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 320, 256, 3, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 2, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 2, 3 }, { 4, 1 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 3, 1 }, { 4, 1 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 3, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 3, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 2, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 2, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 3, 1 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 3, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 1, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, 3, 0 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, 3, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 1, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 2, 1 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 2, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 1, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 1, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 3, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, 3, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, 2, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, 2, 3 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 2, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 2, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 3, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 1, 3 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 2, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 3, 1 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 3, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 576, 512, 2, 0 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 576, 512, 2, 1 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q4_1, 576, 512, 3, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 32, 32, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, 3, 2 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, 1, 0 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, 1, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, 2, 4 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, 3, 4 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 1, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 1 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 2, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 3, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 1, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 3, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 3, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 3, 0 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 3, 3 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, 2, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, 2, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, 3, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 320, 256, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 320, 256, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 320, 256, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 2, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 1, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 1, 4 }, { 1, 1 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 2, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 1, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 3, 3 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, 1, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 64, 64, 1, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 1, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 2, 4 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 3, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, 3, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 256, 256, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 2, 0 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 1, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 2, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 2, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 3, 1 }, { 4, 1 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 3, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q8_0, 576, 512, 2, 1 }, { 4, 4 } }, }; -static enum ggml_metal_device_id fa_vec_family_representative(int gpu_family) { - switch (gpu_family) { - case 9: return GGML_METAL_DEVICE_M4_MAX; - default: return GGML_METAL_DEVICE_GENERIC; - } -} - static bool g_override_set = false; static fa_vec_cfg_t g_override_cfg = { 1, 4 }; @@ -3853,7 +1134,7 @@ static const fa_vec_cfg_t * find_cfg(const fa_vec_entry_t * tbl, size_t n, const return nullptr; } -fa_vec_cfg_t fa_vec_pick(enum ggml_metal_device_id device_id, int gpu_family, int dtype, int dk, int dv, int64_t ne11, int64_t ne01) { +fa_vec_cfg_t fa_vec_pick(int gpu_family, int dtype, int dk, int dv, int64_t ne11, int64_t ne01) { if (g_override_set) { return g_override_cfg; } @@ -3864,38 +1145,25 @@ fa_vec_cfg_t fa_vec_pick(enum ggml_metal_device_id device_id, int gpu_family, in if (ne11_b == 0) { return baseline; // short KV: attention is a small slice of the step, left to baseline } + const int ne01_b = fa_vec_ne01_bucket(ne01); fa_vec_key_t k{}; - k.dtype = (int8_t) dtype; - k.dk = (int16_t) dk; - k.dv = (int16_t) dv; - - // exact bucket, then the ne01 domain default (ne11 collapsed); tried under each device tier - auto lookup = [&](enum ggml_metal_device_id dev) -> const fa_vec_cfg_t * { - k.device_id = (int8_t) dev; - k.ne11_b = (int8_t) ne11_b; - k.ne01_b = (int8_t) ne01_b; - if (auto * c = find_cfg(fa_vec_tuned_table, std::size(fa_vec_tuned_table), k)) { - return c; - } - k.ne11_b = FA_VEC_NE11_DEFAULT; - k.ne01_b = (ne01_b == 0) ? FA_VEC_DOMAIN_DECODE : FA_VEC_DOMAIN_BATCH; - return find_cfg(fa_vec_tuned_table, std::size(fa_vec_tuned_table), k); - }; + k.family = (int8_t) gpu_family; + k.dtype = (int8_t) dtype; + k.dk = (int16_t) dk; + k.dv = (int16_t) dv; + k.ne11_b = (int8_t) ne11_b; + k.ne01_b = (int8_t) ne01_b; - if (auto * c = lookup(device_id)) { + // exact bucket, then the ne01 domain default (ne11 collapsed) + if (auto * c = find_cfg(fa_vec_tuned_table, std::size(fa_vec_tuned_table), k)) { return *c; } - - // family fallback: retry under the family's representative SKU; none -> baseline - if (gpu_family > 0) { - const enum ggml_metal_device_id rep = fa_vec_family_representative(gpu_family); - if (rep != GGML_METAL_DEVICE_GENERIC) { - if (auto * c = lookup(rep)) { - return *c; - } - } + k.ne11_b = FA_VEC_NE11_DEFAULT; + k.ne01_b = (ne01_b == 0) ? FA_VEC_DOMAIN_DECODE : FA_VEC_DOMAIN_BATCH; + if (auto * c = find_cfg(fa_vec_tuned_table, std::size(fa_vec_tuned_table), k)) { + return *c; } return baseline; diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.h b/ggml/src/ggml-metal/ggml-metal-tuning.h index 640ce53efbab..003b4d6bbbed 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.h +++ b/ggml/src/ggml-metal/ggml-metal-tuning.h @@ -1,6 +1,5 @@ #pragma once -#include "ggml-metal-device.h" // enum ggml_metal_device_id #include "ggml.h" #include <cstdint> @@ -32,7 +31,7 @@ constexpr int8_t FA_VEC_DOMAIN_DECODE = 0; // ne01 == 1 constexpr int8_t FA_VEC_DOMAIN_BATCH = 1; // ne01 >= 2 struct fa_vec_key_t { - int8_t device_id; + int8_t family; int8_t dtype; int16_t dk; int16_t dv; @@ -70,8 +69,7 @@ void fa_vec_set_override(fa_vec_cfg_t cfg); void fa_vec_clear_override(); fa_vec_cfg_t fa_vec_baseline_cfg(int dk, int dv); -// device_id selects a per-SKU row; on a miss, gpu_family (0 if unknown) maps to a representative -// SKU and the table is retried. No match -> baseline. -fa_vec_cfg_t fa_vec_pick(enum ggml_metal_device_id device_id, int gpu_family, int dtype, int dk, int dv, int64_t ne11, int64_t ne01); +// Keyed by Apple GPU family; an untuned family matches no row and gets the baseline. +fa_vec_cfg_t fa_vec_pick(int gpu_family, int dtype, int dk, int dv, int64_t ne11, int64_t ne01); } // namespace ggml_metal_tuning diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 3bd6abd06fdc..c6c8ce836742 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -4,6 +4,7 @@ #include "ggml-backend-impl.h" #include "ggml-metal-device.h" +#include "ggml-metal-fusion.h" #include "ggml-metal-context.h" #include "ggml-metal-ops.h" #include "ggml-metal-tuning.h" @@ -225,6 +226,7 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_ { res += ggml_metal_op_mul_mat_id_extra_tpe(tensor); res += ggml_metal_op_mul_mat_id_extra_ids(tensor); + res += ggml_metal_op_mul_mat_id_extra_amax(tensor); } break; case GGML_OP_FLASH_ATTN_EXT: { @@ -560,7 +562,11 @@ static void ggml_backend_metal_event_wait(ggml_backend_t backend, ggml_backend_e } static void ggml_backend_metal_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) { - GGML_UNUSED(params); + GGML_ASSERT(params && params->add_alloc_dep); + + // keep the MoE weighted-reduction inputs alive until the fused output so the + // allocator cannot reuse them while the fused kernel is still reading them + ggml_metal_fusion_add_alloc_deps(params->user_data, params->add_alloc_dep, cgraph); ggml_metal_t ctx = (ggml_metal_t)backend->context; @@ -906,6 +912,30 @@ static const char * ggml_backend_metal_tuning_device_token(ggml_backend_dev_t de return ggml_metal_device_id_token(ggml_metal_device_get_props(ctx_dev)->device_id); } +// generic fusion debugging API (ad-hoc proc-address mechanism): the test resolves the device +// fusion context once and passes that opaque handle to the rest of the functions +typedef void * ggml_backend_fusion_t; + +static ggml_backend_fusion_t ggml_backend_metal_fusion_get(ggml_backend_dev_t dev) { + return ggml_metal_device_get_fusion_info((ggml_metal_device_t)dev->context); +} + +static void ggml_backend_metal_fusion_stats_init(ggml_backend_fusion_t finfo) { + ggml_metal_fusion_info_stats_init((struct ggml_metal_fusion_info *) finfo); +} + +static void ggml_backend_metal_fusion_stats_reset(ggml_backend_fusion_t finfo) { + ggml_metal_fusion_info_stats_reset((struct ggml_metal_fusion_info *) finfo); +} + +static int ggml_backend_metal_fusion_stats_get(ggml_backend_fusion_t finfo, const char ** labels, uint64_t * counts, int n) { + return ggml_metal_fusion_info_stats_get((struct ggml_metal_fusion_info *) finfo, labels, counts, n); +} + +static void ggml_backend_metal_fusion_set_enabled(ggml_backend_fusion_t finfo, bool enabled) { + ggml_metal_fusion_info_set_enabled((struct ggml_metal_fusion_info *) finfo, enabled); +} + static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) { if (strcmp(name, "ggml_backend_get_features") == 0) { return (void *)ggml_backend_metal_get_features; @@ -928,6 +958,23 @@ static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const if (strcmp(name, "ggml_backend_metal_tuning_device_token") == 0) { return (void *)ggml_backend_metal_tuning_device_token; } + // generic fusion debugging API (ad-hoc proc-address mechanism, not part of the official + // ggml backend interface yet; a backend that adopts it exports these exact names) + if (strcmp(name, "ggml_backend_fusion_get") == 0) { + return (void *)ggml_backend_metal_fusion_get; + } + if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) { + return (void *)ggml_backend_metal_fusion_stats_init; + } + if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) { + return (void *)ggml_backend_metal_fusion_stats_reset; + } + if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) { + return (void *)ggml_backend_metal_fusion_stats_get; + } + if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) { + return (void *)ggml_backend_metal_fusion_set_enabled; + } return NULL; diff --git a/ggml/src/ggml-metal/kernels/argsort.metal b/ggml/src/ggml-metal/kernels/argsort.metal index e81d194c339f..5231b8395c9a 100644 --- a/ggml/src/ggml-metal/kernels/argsort.metal +++ b/ggml/src/ggml-metal/kernels/argsort.metal @@ -1,5 +1,11 @@ #include "common.h" +constant bool FC_topk_moe_with_norm [[function_constant(FC_TOPK_MOE + 0)]]; +constant int FC_topk_moe_n_expert [[function_constant(FC_TOPK_MOE + 1)]]; +constant int FC_topk_moe_top_k [[function_constant(FC_TOPK_MOE + 2)]]; + +constant int FC_moe_reduce_n_expert_used [[function_constant(FC_MOE_REDUCE + 0)]]; + // bitonic sort implementation following the CUDA kernels as reference typedef void (argsort_t)( constant ggml_metal_kargs_argsort & args, @@ -335,3 +341,139 @@ kernel void kernel_top_k_f32_i32( } } } + +// fused SOFT_MAX + top-k + GET_ROWS (+ optional norm/scale) for MoE routing. +// One SIMDgroup handles one token row; n_expert is limited to 1024 by the host. +kernel void kernel_topk_moe_f32( + constant ggml_metal_kargs_topk_moe & args, + device const char * src0, + device float * weights, + device int32_t * ids, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]]) { + const int row = (int) tgpig.x; + if (row >= args.ne01) { + return; + } + + const int n_expert = FC_topk_moe_n_expert; + const int top_k = FC_topk_moe_top_k; + const int lane = (int) tiisg; + const int n_per_lane = (n_expert + 31) / 32; + + device const float * logits_row = (device const float *) (src0 + row * args.nb01); + device float * weights_row = weights + row * top_k; + device int32_t * ids_row = ids + row * (args.nb1_ids / sizeof(int32_t)); + + float wt[32]; + float output_weights[32]; + FOR_UNROLL (int i = 0; i < 32; ++i) { + wt[i] = -INFINITY; + output_weights[i] = 0.0f; + } + + for (int i = lane; i < n_expert; i += 32) { + const float v = logits_row[i]; + wt[i / 32] = isnan(v) ? -FLT_MAX : v; + } + + // softmax over the expert logits + float max_val = -INFINITY; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + max_val = max(max_val, wt[i]); + } + max_val = simd_max(max_val); + + float sum_val = 0.0f; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + wt[i] = exp(wt[i] - max_val); + sum_val += wt[i]; + } + sum_val = simd_sum(sum_val); + + const float inv_sum = 1.0f / sum_val; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + wt[i] *= inv_sum; + } + + float wt_sum = 0.0f; + + for (int k = 0; k < top_k; ++k) { + float best_val = -INFINITY; + int best_expert = -1; + + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + const int expert = lane + i * 32; + if (expert < n_expert && (wt[i] > best_val || (wt[i] == best_val && expert < best_expert))) { + best_val = wt[i]; + best_expert = expert; + } + } + + FOR_UNROLL (int mask = 16; mask > 0; mask >>= 1) { + const float val = simd_shuffle_xor(best_val, mask); + const int expert = simd_shuffle_xor(best_expert, mask); + if (val > best_val || (val == best_val && expert < best_expert)) { + best_val = val; + best_expert = expert; + } + } + + if ((best_expert & 31) == lane) { + wt[best_expert / 32] = -INFINITY; + } + + if ((k & 31) == lane) { + output_weights[k / 32] = best_val; + } + + if ((best_expert & 31) == lane) { + ids_row[k] = best_expert; + if (FC_topk_moe_with_norm) { + wt_sum += best_val; + } + } + } + + if (FC_topk_moe_with_norm) { + wt_sum = simd_sum(wt_sum); + wt_sum = max(wt_sum, args.clamp); + const float inv = 1.0f / wt_sum; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + output_weights[i] *= inv; + } + } + + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + const int idx = i * 32 + lane; + if (idx < top_k) { + weights_row[idx] = output_weights[i] * args.scale; + } + } +} + +// fused MoE expert weighting + reduction: weighted = sum(experts[e] * weights[e]). +// The host guarantees all tensors are contiguous F32. +kernel void kernel_moe_reduce_f32( + constant ggml_metal_kargs_moe_reduce & args, + device const float * experts, + device const float * weights, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int64_t token = tgpig.x; + const int64_t col = (int64_t) tgpig.y * ntg.x + tpitg.x; + if (token >= args.ne02 || col >= args.ne00) { + return; + } + + const int n_expert_used = FC_moe_reduce_n_expert_used; + + const int64_t base = token * (int64_t) n_expert_used * args.ne00 + col; + float sum = 0.0f; + FOR_UNROLL (int e = 0; e < n_expert_used; ++e) { + sum += experts[base + e * args.ne00] * weights[token * n_expert_used + e]; + } + dst[token * args.ne00 + col] = sum; +} diff --git a/ggml/src/ggml-metal/kernels/fa.metal b/ggml/src/ggml-metal/kernels/fa.metal index d0e928d732cc..f26d493d5424 100644 --- a/ggml/src/ggml-metal/kernels/fa.metal +++ b/ggml/src/ggml-metal/kernels/fa.metal @@ -142,7 +142,7 @@ kernel void kernel_flash_attn_ext_blk( const int32_t i1 = tgpig[1]; const int32_t i0 = tgpig[0]; - char res = i0*C + C > args.ne30 ? 1 : 0; + char res = i0*C + C > args.ne30 || i1*Q + Q > args.ne31 ? 1 : 0; device const half * mask_src = (device const half *) (mask + (i1*Q)*args.nb31 + i2*args.nb32 + i3*args.nb33) + i0*C + tiisg; @@ -930,6 +930,7 @@ template [[host_name("kernel_flash_attn_ext_f32_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_f32_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 72, 72>; template [[host_name("kernel_flash_attn_ext_f32_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 80, 80>; template [[host_name("kernel_flash_attn_ext_f32_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 96, 96>; +template [[host_name("kernel_flash_attn_ext_f32_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 96, 64>; template [[host_name("kernel_flash_attn_ext_f32_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 112, 112>; template [[host_name("kernel_flash_attn_ext_f32_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 128, 128>; template [[host_name("kernel_flash_attn_ext_f32_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_F32, float4x4, 1, dequantize_f32, float4x4, 1, dequantize_f32, 192, 192>; @@ -946,6 +947,7 @@ template [[host_name("kernel_flash_attn_ext_f16_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_f16_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 72, 72>; template [[host_name("kernel_flash_attn_ext_f16_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 80, 80>; template [[host_name("kernel_flash_attn_ext_f16_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 96, 96>; +template [[host_name("kernel_flash_attn_ext_f16_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 96, 64>; template [[host_name("kernel_flash_attn_ext_f16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 112, 112>; template [[host_name("kernel_flash_attn_ext_f16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 128, 128>; template [[host_name("kernel_flash_attn_ext_f16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, half4x4, 1, dequantize_f16, half4x4, 1, dequantize_f16, 192, 192>; @@ -963,6 +965,7 @@ template [[host_name("kernel_flash_attn_ext_bf16_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_bf16_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 72, 72>; template [[host_name("kernel_flash_attn_ext_bf16_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 80, 80>; template [[host_name("kernel_flash_attn_ext_bf16_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 96, 96>; +template [[host_name("kernel_flash_attn_ext_bf16_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 96, 64>; template [[host_name("kernel_flash_attn_ext_bf16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 112, 112>; template [[host_name("kernel_flash_attn_ext_bf16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 128, 128>; template [[host_name("kernel_flash_attn_ext_bf16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES_BF, bfloat4x4, 1, dequantize_bf16, bfloat4x4, 1, dequantize_bf16, 192, 192>; @@ -980,6 +983,7 @@ template [[host_name("kernel_flash_attn_ext_q4_0_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q4_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 72, 72>; template [[host_name("kernel_flash_attn_ext_q4_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 80, 80>; template [[host_name("kernel_flash_attn_ext_q4_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 96, 96>; +template [[host_name("kernel_flash_attn_ext_q4_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 96, 64>; template [[host_name("kernel_flash_attn_ext_q4_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 112, 112>; template [[host_name("kernel_flash_attn_ext_q4_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 128, 128>; template [[host_name("kernel_flash_attn_ext_q4_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_0, 2, dequantize_q4_0, block_q4_0, 2, dequantize_q4_0, 192, 192>; @@ -996,6 +1000,7 @@ template [[host_name("kernel_flash_attn_ext_q4_1_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q4_1_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 72, 72>; template [[host_name("kernel_flash_attn_ext_q4_1_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 80, 80>; template [[host_name("kernel_flash_attn_ext_q4_1_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 96, 96>; +template [[host_name("kernel_flash_attn_ext_q4_1_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 96, 64>; template [[host_name("kernel_flash_attn_ext_q4_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 112, 112>; template [[host_name("kernel_flash_attn_ext_q4_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 128, 128>; template [[host_name("kernel_flash_attn_ext_q4_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q4_1, 2, dequantize_q4_1, block_q4_1, 2, dequantize_q4_1, 192, 192>; @@ -1012,6 +1017,7 @@ template [[host_name("kernel_flash_attn_ext_q5_0_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q5_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 72, 72>; template [[host_name("kernel_flash_attn_ext_q5_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 80, 80>; template [[host_name("kernel_flash_attn_ext_q5_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 96, 96>; +template [[host_name("kernel_flash_attn_ext_q5_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 96, 64>; template [[host_name("kernel_flash_attn_ext_q5_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 112, 112>; template [[host_name("kernel_flash_attn_ext_q5_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 128, 128>; template [[host_name("kernel_flash_attn_ext_q5_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_0, 2, dequantize_q5_0, block_q5_0, 2, dequantize_q5_0, 192, 192>; @@ -1028,6 +1034,7 @@ template [[host_name("kernel_flash_attn_ext_q5_1_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q5_1_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 72, 72>; template [[host_name("kernel_flash_attn_ext_q5_1_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 80, 80>; template [[host_name("kernel_flash_attn_ext_q5_1_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 96, 96>; +template [[host_name("kernel_flash_attn_ext_q5_1_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 96, 64>; template [[host_name("kernel_flash_attn_ext_q5_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 112, 112>; template [[host_name("kernel_flash_attn_ext_q5_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 128, 128>; template [[host_name("kernel_flash_attn_ext_q5_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q5_1, 2, dequantize_q5_1, block_q5_1, 2, dequantize_q5_1, 192, 192>; @@ -1044,6 +1051,7 @@ template [[host_name("kernel_flash_attn_ext_q8_0_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q8_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 72, 72>; template [[host_name("kernel_flash_attn_ext_q8_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 80, 80>; template [[host_name("kernel_flash_attn_ext_q8_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 96, 96>; +template [[host_name("kernel_flash_attn_ext_q8_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 96, 64>; template [[host_name("kernel_flash_attn_ext_q8_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 112, 112>; template [[host_name("kernel_flash_attn_ext_q8_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 128, 128>; template [[host_name("kernel_flash_attn_ext_q8_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext<FA_TYPES, block_q8_0, 2, dequantize_q8_0, block_q8_0, 2, dequantize_q8_0, 192, 192>; @@ -1905,6 +1913,29 @@ template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96")]] kernel flas template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 96, 96, 4, 2>; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 96, 96, 4, 4>; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES_F32, float4, 1, dequantize_f32_t4, float4, 1, dequantize_f32_t4, 96, 64, 4>; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 96, 64, 4>; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 96, 64, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 96, 64, 4, 4>; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, bfloat4, 1, dequantize_bf16_t4, bfloat4, 1, dequantize_bf16_t4, 96, 64, 4>; +#endif +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 96, 64, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 96, 64, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_0, 8, dequantize_q4_0_t4, block_q4_0, 8, dequantize_q4_0_t4, 96, 64, 4, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 96, 64, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 96, 64, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q4_1, 8, dequantize_q4_1_t4, block_q4_1, 8, dequantize_q4_1_t4, 96, 64, 4, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 96, 64, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 96, 64, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_0, 8, dequantize_q5_0_t4, block_q5_0, 8, dequantize_q5_0_t4, 96, 64, 4, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 96, 64, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 96, 64, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q5_1, 8, dequantize_q5_1_t4, block_q5_1, 8, dequantize_q5_1_t4, 96, 64, 4, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 96, 64, 4>; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 96, 64, 4, 2>; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, block_q8_0, 8, dequantize_q8_0_t4, block_q8_0, 8, dequantize_q8_0_t4, 96, 64, 4, 4>; + template [[host_name("kernel_flash_attn_ext_vec_f32_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES_F32, float4, 1, dequantize_f32_t4, float4, 1, dequantize_f32_t4, 128, 128, 1>; template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 128, 128, 1>; template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec<FA_TYPES, half4, 1, dequantize_f16_t4, half4, 1, dequantize_f16_t4, 128, 128, 2, 1>; diff --git a/ggml/src/ggml-metal/kernels/gated_delta_net.metal b/ggml/src/ggml-metal/kernels/gated_delta_net.metal index 8422d8e29f8b..5e4861ece360 100644 --- a/ggml/src/ggml-metal/kernels/gated_delta_net.metal +++ b/ggml/src/ggml-metal/kernels/gated_delta_net.metal @@ -15,6 +15,7 @@ kernel void kernel_gated_delta_net_impl( device const char * b, device const char * s, device char * dst, + device char * dst_fuse, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -65,6 +66,12 @@ kernel void kernel_gated_delta_net_impl( // per-(seq,head) offset within a slot const uint state_out_base = (i23*args.ne21 + i21)*S_v*S_v + i20*S_v; + // when fused with the cache cpy, write the snapshots straight into the cache buffer using + // the slot stride; otherwise append them after the attn scores (nb_out == 0) + const bool fused = args.nb_out > 0; + const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + attn_size; + const uint slot_stride = fused ? (uint)args.nb_out : state_size_per_snap; + for (short t = 0; t < args.ne22; t++) { float s_k = 0.0f; @@ -116,7 +123,7 @@ kernel void kernel_gated_delta_net_impl( if (K > 1) { const int target_slot = (int)args.ne22 - 1 - (int)t; if (target_slot >= 0 && target_slot < (int)K) { - device float * dst_state = (device float *) (dst) + attn_size + (uint)target_slot * state_size_per_snap + state_out_base; + device float * dst_state = (device float *)state_out + (uint)target_slot * slot_stride + state_out_base; FOR_UNROLL (short j = 0; j < NSG; j++) { const short is = tx*NSG + j; dst_state[is] = ls[j]; @@ -126,7 +133,7 @@ kernel void kernel_gated_delta_net_impl( } if (K == 1) { - device float * dst_state = (device float *) (dst) + attn_size + state_out_base; + device float * dst_state = (device float *)state_out + state_out_base; FOR_UNROLL (short j = 0; j < NSG; j++) { const short is = tx*NSG + j; dst_state[is] = ls[j]; @@ -158,6 +165,7 @@ kernel void kernel_gated_delta_net_impl( device const char * b, device const char * s, device char * dst, + device char * dst_fuse, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -230,7 +238,13 @@ kernel void kernel_gated_delta_net_impl( dst_attn += args.ne21*S_v; } - device float * dst_state = (device float *) (dst) + args.ne23*args.ne22*args.ne21*S_v + (i23*args.ne21 + i21)*S_v*S_v + i20; + // when fused with the cache cpy, write the snapshots straight into the cache buffer using + // the slot stride; otherwise append them after the attn scores (nb_out == 0) + const bool fused = args.nb_out > 0; + const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + args.ne23*args.ne22*args.ne21*S_v; + const uint slot_stride = fused ? (uint)args.nb_out : S_v*S_v; + + device float * dst_state = (device float *)state_out + (i23*args.ne21 + i21)*slot_stride + i20; device T * dstt_state = (device T *) (dst_state); FOR_UNROLL (short j = 0; j < NSG; j++) { diff --git a/ggml/src/ggml-metal/kernels/misc.metal b/ggml/src/ggml-metal/kernels/misc.metal index 11104b4d8d12..279d69f8fe96 100644 --- a/ggml/src/ggml-metal/kernels/misc.metal +++ b/ggml/src/ggml-metal/kernels/misc.metal @@ -374,10 +374,10 @@ template [[host_name("kernel_snake_f16")]] kernel void kernel_snake<half>(const template [[host_name("kernel_snake_bf16")]] kernel void kernel_snake<bfloat>(constant ggml_metal_kargs_snake &, device const bfloat *, device const float *, device const float *, device bfloat *, uint, uint, uint); #endif -template<int N> -kernel void kernel_fwht_f32( +template<int N, typename src_t> +kernel void kernel_fwht( constant ggml_metal_kargs_fwht & args, - device const float * src, + device const src_t * src, device float * dst, uint3 tgpig[[threadgroup_position_in_grid]], ushort sgitg[[simdgroup_index_in_threadgroup]], @@ -402,13 +402,13 @@ kernel void kernel_fwht_f32( float reg[NE]; for (int i = 0; i < NE; i++) { - reg[i] = src[i*NW + lane]*scale; + reg[i] = float(src[i*NW + lane])*scale; } for (int i = 1; i < NW; i *= 2) { for (int j = 0; j < NE; j++) { const float val = reg[j]; const float val2 = simd_shuffle_xor(val, i); - reg[j] = (lane & i) == 0 ? val2 + val : val2 - val; + reg[j] = val2 - val + 2*((lane & i) == 0)*val; } } @@ -429,12 +429,20 @@ kernel void kernel_fwht_f32( } } -typedef decltype(kernel_fwht_f32<64>) kernel_fwht_t; +typedef decltype(kernel_fwht<64, float>) kernel_fwht_f32_t; +typedef decltype(kernel_fwht<64, half>) kernel_fwht_f16_t; -template [[host_name("kernel_fwht_f32_64")]] kernel kernel_fwht_t kernel_fwht_f32<64>; -template [[host_name("kernel_fwht_f32_128")]] kernel kernel_fwht_t kernel_fwht_f32<128>; -template [[host_name("kernel_fwht_f32_256")]] kernel kernel_fwht_t kernel_fwht_f32<256>; -template [[host_name("kernel_fwht_f32_512")]] kernel kernel_fwht_t kernel_fwht_f32<512>; +template [[host_name("kernel_fwht_f32_64")]] kernel kernel_fwht_f32_t kernel_fwht<64, float>; +template [[host_name("kernel_fwht_f32_128")]] kernel kernel_fwht_f32_t kernel_fwht<128, float>; +template [[host_name("kernel_fwht_f32_256")]] kernel kernel_fwht_f32_t kernel_fwht<256, float>; +template [[host_name("kernel_fwht_f32_512")]] kernel kernel_fwht_f32_t kernel_fwht<512, float>; + +template [[host_name("kernel_fwht_f16_64")]] kernel kernel_fwht_f16_t kernel_fwht<64, half>; +template [[host_name("kernel_fwht_f16_128")]] kernel kernel_fwht_f16_t kernel_fwht<128, half>; +template [[host_name("kernel_fwht_f16_256")]] kernel kernel_fwht_f16_t kernel_fwht<256, half>; +template [[host_name("kernel_fwht_f16_512")]] kernel kernel_fwht_f16_t kernel_fwht<512, half>; + +constant int FC_dsv4_hc_n_hc [[function_constant(FC_DSV4_HC + 0)]]; kernel void kernel_dsv4_hc_comb_f32( constant ggml_metal_kargs_dsv4_hc_comb & args, @@ -506,32 +514,87 @@ kernel void kernel_dsv4_hc_pre_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]], ushort3 ntg[[threads_per_threadgroup]]) { + const int it = tgpig.y; + const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg; + + if (i0 >= args.n_embd) { + return; + } + + device const char * xb = x + i0*args.nb_x0 + it*args.nb_x2; + float result = 0.0f; + FOR_UNROLL (int ih = 0; ih < FC_dsv4_hc_n_hc; ++ih) { + const float xv = *(device const float *) (xb + ih*args.nb_x1); + const float wv = *(device const float *) (weights + ih*args.nb_w0 + it*args.nb_w1); + result = fma(xv, wv, result); + } + + *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = args.scale*result; +} + +kernel void kernel_dsv4_hc_pre_gated_f32( + constant ggml_metal_kargs_dsv4_hc_pre & args, + device const char * x, + device const char * gate, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int it = tgpig.y; + const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg; + + if (i0 >= args.n_embd) { + return; + } + + device const char * xb = x + i0*args.nb_x0 + it*args.nb_x2; + device const char * gb = gate + i0*args.nb_w0 + it*args.nb_w2; + float result = 0.0f; + FOR_UNROLL (int ih = 0; ih < FC_dsv4_hc_n_hc; ++ih) { + const float g = 1.0f/(1.0f + exp(-*(device const float *) (gb + ih*args.nb_w1))); + const float xv = *(device const float *) (xb + ih*args.nb_x1); + result = fma(xv, g, result); + } + + *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = args.scale*result; +} + +kernel void kernel_dsv4_hc_post_nocomb_f32( + constant ggml_metal_kargs_dsv4_hc_post & args, + device const char * x, + device const char * residual, + device const char * post, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { constexpr ushort hc = 4; const int it = tgpig.y; const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg; - float weight_lane = 0.0f; + float post_lane = 0.0f; if (tiisg < hc) { - weight_lane = *(device const float *) (weights + tiisg*args.nb_w0 + it*args.nb_w1); + post_lane = *(device const float *) (post + tiisg*args.nb_p0 + it*args.nb_p1); } - float w[hc]; - FOR_UNROLL (ushort ih = 0; ih < hc; ++ih) { - w[ih] = simd_shuffle(weight_lane, ih); + float post_reg[hc]; + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + post_reg[idst] = simd_shuffle(post_lane, idst); } if (i0 >= args.n_embd) { return; } - device const char * xb = x + i0*args.nb_x0 + it*args.nb_x2; - float result = 0.0f; - FOR_UNROLL (ushort ih = 0; ih < hc; ++ih) { - result = fma(*(device const float *) (xb + ih*args.nb_x1), w[ih], result); + const float xv = *(device const float *) (x + i0*args.nb_x0 + it*args.nb_x1); + device const char * rb = residual + i0*args.nb_r0 + it*args.nb_r2; + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + const float rv = *(device const float *) (rb + idst*args.nb_r1); + *(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = xv*post_reg[idst] + rv; } - - *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = result; } kernel void kernel_dsv4_hc_post_f32( diff --git a/ggml/src/ggml-metal/kernels/mul_mm.metal b/ggml/src/ggml-metal/kernels/mul_mm.metal index ee848eed6d6a..a25838f92647 100644 --- a/ggml/src/ggml-metal/kernels/mul_mm.metal +++ b/ggml/src/ggml-metal/kernels/mul_mm.metal @@ -7,6 +7,7 @@ constant short FC_mul_mm_ne12 [[function_constant(FC_MUL_MM + 2)]]; constant short FC_mul_mm_ne13 [[function_constant(FC_MUL_MM + 3)]]; constant short FC_mul_mm_r2 [[function_constant(FC_MUL_MM + 4)]]; constant short FC_mul_mm_r3 [[function_constant(FC_MUL_MM + 5)]]; +constant bool FC_mul_mm_id_amax [[function_constant(FC_MUL_MM + 6)]]; // each block_q contains 16*nl weights #ifdef GGML_METAL_HAS_TENSOR @@ -413,6 +414,85 @@ kernel void kernel_mul_mm_id_map0( tpe_u32[ide] = n_all; } +kernel void kernel_mul_mm_id_amax_part_f32( + constant ggml_metal_kargs_mul_mm_id_amax & args, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort ntg[[threads_per_threadgroup]]) { + const int nrow = args.ne01*args.ne02; + + float lmax = 0.0f; + + for (int ir = tgpig; ir < nrow; ir += N_MM_NPART_AMAX) { + const int i01 = ir % args.ne01; + const int i02 = ir / args.ne01; + + device const float * row = (device const float *) (src1 + i02*args.nb02 + i01*args.nb01); + + for (int i00 = tiitg; i00 < args.ne00; i00 += ntg) { + lmax = max(lmax, fabs(row[i00])); + } + } + + float amax = simd_max(lmax); + + threadgroup float * shared_amax = (threadgroup float *) shmem; + + if (ntg > N_SIMDWIDTH) { + if (sgitg == 0) { + shared_amax[tiisg] = 0.0f; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shared_amax[sgitg] = amax; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + amax = shared_amax[tiisg]; + amax = simd_max(amax); + } + + if (tiitg == 0) { + ((device float *) (dst + 8))[tgpig] = amax; + } +} + +kernel void kernel_mul_mm_id_amax_f32( + device char * dst, + ushort tiitg[[thread_index_in_threadgroup]]) { + device const float * part = (device const float *) (dst + 8); + + float amax = 0.0f; + + for (int i = tiitg; i < N_MM_NPART_AMAX; i += N_SIMDWIDTH) { + amax = max(amax, part[i]); + } + + amax = simd_max(amax); + + if (tiitg == 0) { + // leave a comfortable margin below the f16 max of 65504 + float scale = 1.0f; + + // isfinite: src1 already inf/nan is not ours to fix - keep the + // scale at 1.0 instead of turning it into a different failure + if (isfinite(amax) && amax > 32768.0f) { + scale = exp2(ceil(log2(amax)) - 15.0f); + } + + device float * d = (device float *) dst; + + d[0] = 1.0f/scale; // exact: scale is a power of two + d[1] = scale; + } +} + typedef decltype(kernel_mul_mm_id_map0<1>) kernel_mul_mm_id_map0_t; template [[host_name("kernel_mul_mm_id_map0_ne20_1" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<1>; @@ -433,6 +513,7 @@ kernel void kernel_mul_mm_id( device const char * htpe, device const char * hids, device char * dst, + device const char * amax, threadgroup char * shmem [[threadgroup(0)]], uint3 tgpig[[threadgroup_position_in_grid]], ushort tiitg[[thread_index_in_threadgroup]], @@ -496,6 +577,17 @@ kernel void kernel_mul_mm_id( + args.nb11*i11 + args.nb10*iy); + // skip the upper half of the token tile when the expert did not fill it + constexpr short NR1H = NR1/2; + + const bool has_hi = nr1 > NR1H; + + const short lb1 = (short) tiitg/NL1; // 0 .. NR1-1, this thread's row of the B tile + + // power-of-two rescaling + const float s1_inv = FC_mul_mm_id_amax ? ((device const float *) amax)[0] : 1.0f; + const float s1_scale = FC_mul_mm_id_amax ? ((device const float *) amax)[1] : 1.0f; + #ifndef GGML_METAL_HAS_TENSOR S0_8x8 ma[4]; S1_8x8 mb[2]; @@ -505,15 +597,22 @@ kernel void kernel_mul_mm_id( for (short i = 0; i < 8; i++){ mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f); } + + // simdgroups 2,3 own rows NR1H..NR1-1 + const bool sg_active = has_hi || sgitg < 2; #else - auto tA = tensor<threadgroup S0, dextents<int32_t, 2>, tensor_inline>(sa, dextents<int32_t, 2>(NK, NR0)); - auto tB = tensor<threadgroup S1, dextents<int32_t, 2>, tensor_inline>(sb, dextents<int32_t, 2>(NR1, NK )); + auto tA = tensor<threadgroup S0, dextents<int32_t, 2>, tensor_inline>(sa, dextents<int32_t, 2>(NK, NR0)); + + // sb is [NR1][NK] row-major + auto tB0 = tensor<threadgroup S1, dextents<int32_t, 2>, tensor_inline>(sb, dextents<int32_t, 2>(NK, NR1H)); + auto tB1 = tensor<threadgroup S1, dextents<int32_t, 2>, tensor_inline>(sb + NR1H*NK, dextents<int32_t, 2>(NK, NR1H)); mpp::tensor_ops::matmul2d< - mpp::tensor_ops::matmul2d_descriptor(NR1, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), + mpp::tensor_ops::matmul2d_descriptor(NR1H, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), execution_simdgroups<4>> mm; - auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); + auto cT0 = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB0), float>(); + auto cT1 = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB1), float>(); #endif for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { @@ -572,7 +671,7 @@ kernel void kernel_mul_mm_id( const short ib = 4*sx + sy; - *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; + *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) (*((device T1 *) y + i) * (T1) s1_inv) : 0; } } else { const short sx = (tiitg%NL1); @@ -585,7 +684,7 @@ kernel void kernel_mul_mm_id( const short ib = 4*sx + sy; - *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)(*((device T1_2x4 *) y)); + *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)((*((device T1_2x4 *) y)) * (T1) s1_inv); } #else // load data and store to threadgroup memory @@ -633,7 +732,7 @@ kernel void kernel_mul_mm_id( //const short lx = (tiitg/NL1)%8; //const short ly = i; - *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; + *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) (*((device T1 *) y + i) * (T1) s1_inv) : 0; } } else { const short sx = (tiitg%NL1); @@ -644,7 +743,7 @@ kernel void kernel_mul_mm_id( //const short lx = (tiitg/NL1)%8; //const short ly = i; - *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)(*((device T1_2x4 *) y)); + *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)((*((device T1_2x4 *) y)) * (T1) s1_inv); } #endif @@ -656,37 +755,45 @@ kernel void kernel_mul_mm_id( threadgroup_barrier(mem_flags::mem_threadgroup); #ifndef GGML_METAL_HAS_TENSOR - // load matrices from threadgroup memory and conduct outer products - threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); - threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); + if (sg_active) { + // load matrices from threadgroup memory and conduct outer products + threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); + threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); - FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { - simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 4; i++) { - simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); - } + FOR_UNROLL (short i = 0; i < 4; i++) { + simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); + } - simdgroup_barrier(mem_flags::mem_none); + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 2; i++) { - simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); - } + FOR_UNROLL (short i = 0; i < 2; i++) { + simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); + } - simdgroup_barrier(mem_flags::mem_none); + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 8; i++){ - simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); - } + FOR_UNROLL (short i = 0; i < 8; i++){ + simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); + } - lsma += 8*64; - lsmb += 4*64; + lsma += 8*64; + lsmb += 4*64; + } } #else - auto sA = tA.slice(0, 0); - auto sB = tB.slice(0, 0); + auto sA = tA.slice(0, 0); + auto sB0 = tB0.slice(0, 0); - mm.run(sB, sA, cT); + mm.run(sB0, sA, cT0); + + if (has_hi) { + auto sB1 = tB1.slice(0, 0); + + mm.run(sB1, sA, cT1); + } #endif } @@ -694,13 +801,20 @@ kernel void kernel_mul_mm_id( threadgroup_barrier(mem_flags::mem_threadgroup); #ifdef GGML_METAL_HAS_TENSOR - auto tC = tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline>(sc, dextents<int32_t, 2>(NR0, NR1)); - cT.store(tC); + auto tC0 = tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline>(sc, dextents<int32_t, 2>(NR0, NR1H)); + cT0.store(tC0); + + if (has_hi) { + auto tC1 = tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline>(sc + NR1H*NR0, dextents<int32_t, 2>(NR0, NR1H)); + cT1.store(tC1); + } #else - threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; + if (sg_active) { + threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; - for (short i = 0; i < 8; i++) { - simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + for (short i = 0; i < 8; i++) { + simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + } } #endif @@ -720,12 +834,12 @@ kernel void kernel_mul_mm_id( int i = tiisg; for (; i < nr0/4; i += 32) { - *(D4 + i) = *(C4 + i); + *(D4 + i) = *(C4 + i) * s1_scale; } i = (4*(nr0/4)) + tiisg; for (; i < nr0; i += 32) { - *(D + i) = *(C + i); + *(D + i) = *(C + i) * s1_scale; } } } diff --git a/ggml/src/ggml-metal/kernels/mul_mv.metal b/ggml/src/ggml-metal/kernels/mul_mv.metal index d1800313ed6e..0d1069f2c88a 100644 --- a/ggml/src/ggml-metal/kernels/mul_mv.metal +++ b/ggml/src/ggml-metal/kernels/mul_mv.metal @@ -213,6 +213,7 @@ constant short FC_mul_mv_nxpsg [[function_constant(FC_MUL_MV + 1)]]; constant short FC_mul_mv_ne12 [[function_constant(FC_MUL_MV + 2)]]; constant short FC_mul_mv_r2 [[function_constant(FC_MUL_MV + 3)]]; constant short FC_mul_mv_r3 [[function_constant(FC_MUL_MV + 4)]]; +constant bool FC_mul_mv_split [[function_constant(FC_MUL_MV + 5)]]; template<typename block_q_type, short NR0, typename args_t> void mul_vec_q_n_f32_impl( @@ -1042,6 +1043,7 @@ template [[host_name("kernel_mul_mv_f16_f16")]] kernel mul_mv_t_t kernel_mul_m #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_mul_mv_bf16_f32")]] kernel mul_mv_t_t kernel_mul_mv_t_t<bfloat, float>; template [[host_name("kernel_mul_mv_bf16_bf16")]] kernel mul_mv_t_t kernel_mul_mv_t_t<bfloat, bfloat>; +template [[host_name("kernel_mul_mv_f32_bf16")]] kernel mul_mv_t_t kernel_mul_mv_t_t<float, bfloat>; #endif template<typename T0, typename T04, typename T1, typename T14, short NR0, typename args_t> @@ -1166,6 +1168,7 @@ template [[host_name("kernel_mul_mv_f16_f16_4")]] kernel mul_mv_t_t_4 kernel_m #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_mul_mv_bf16_f32_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4<bfloat, bfloat4, float, float4>; template [[host_name("kernel_mul_mv_bf16_bf16_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4<bfloat, bfloat4, bfloat, bfloat4>; +template [[host_name("kernel_mul_mv_f32_bf16_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4<float, float4, bfloat, bfloat4>; #endif template<typename T0, typename T1, typename args_t> @@ -1231,6 +1234,7 @@ template [[host_name("kernel_mul_mv_f16_f16_short")]] kernel mul_mv_t_t_short_t #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_mul_mv_bf16_f32_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short<bfloat, float>; template [[host_name("kernel_mul_mv_bf16_bf16_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short<bfloat, bfloat>; +template [[host_name("kernel_mul_mv_f32_bf16_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short<float, bfloat>; #endif template<int nr0, typename args_t> @@ -1888,8 +1892,19 @@ void kernel_mul_mv_iq2_xxs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_xxs * x = (device const block_iq2_xxs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -1897,8 +1912,6 @@ void kernel_mul_mv_iq2_xxs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); { @@ -1911,11 +1924,9 @@ void kernel_mul_mv_iq2_xxs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -1927,7 +1938,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl( device const uint16_t * q2 = xr->qs + 4 * ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; device const uint8_t * aux8 = (device const uint8_t *)q2; const uint32_t aux32 = q2[2] | (q2[3] << 16); @@ -1947,7 +1958,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl( q2 += args.nb01/2; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -1960,6 +1971,23 @@ void kernel_mul_mv_iq2_xxs_f32_impl( } } +template<typename args_t> +void kernel_mul_mv_iq2_xxs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_xxs_f32_impl<N_R0_IQ2_XXS_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_xxs_f32_impl<N_R0_IQ2_XXS, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_xxs_f32")]] kernel void kernel_mul_mv_iq2_xxs_f32( constant ggml_metal_kargs_mul_mv & args, @@ -1970,7 +1998,7 @@ kernel void kernel_mul_mv_iq2_xxs_f32( uint3 tgpig[[threadgroup_position_in_grid]], ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_xxs_f32_impl<N_R0_IQ2_XXS, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_xxs_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template<int nr0, typename args_t> @@ -1996,8 +2024,19 @@ void kernel_mul_mv_iq2_xs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_xs * x = (device const block_iq2_xs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2005,8 +2044,6 @@ void kernel_mul_mv_iq2_xs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 512); { @@ -2019,11 +2056,9 @@ void kernel_mul_mv_iq2_xs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2036,7 +2071,7 @@ void kernel_mul_mv_iq2_xs_f32_impl( device const uint8_t * sc = xr->scales + ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const uint8_t ls1 = sc[0] & 0xf; const uint8_t ls2 = sc[0] >> 4; @@ -2065,7 +2100,7 @@ void kernel_mul_mv_iq2_xs_f32_impl( sc += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2078,6 +2113,23 @@ void kernel_mul_mv_iq2_xs_f32_impl( } } +template<typename args_t> +void kernel_mul_mv_iq2_xs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_xs_f32_impl<N_R0_IQ2_XS_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_xs_f32_impl<N_R0_IQ2_XS, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_xs_f32")]] kernel void kernel_mul_mv_iq2_xs_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2089,9 +2141,10 @@ kernel void kernel_mul_mv_iq2_xs_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_xs_f32_impl<N_R0_IQ2_XS, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_xs_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } +// FC_mul_mv_split: for nb32 < 32 (nb32 divides 32), 32/nb32 threads share each chunk and each takes a slice of the rows template<int nr0, typename args_t> void kernel_mul_mv_iq3_xxs_f32_impl( args_t args, @@ -2115,8 +2168,19 @@ void kernel_mul_mv_iq3_xxs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq3_xxs * x = (device const block_iq3_xxs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2124,8 +2188,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint32_t * svalues = (threadgroup uint32_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); { @@ -2138,11 +2200,9 @@ void kernel_mul_mv_iq3_xxs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2155,7 +2215,7 @@ void kernel_mul_mv_iq3_xxs_f32_impl( device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const uint32_t aux32 = gas[0] | (gas[1] << 16); const float d = db * (0.5f + (aux32 >> 28)); @@ -2177,7 +2237,7 @@ void kernel_mul_mv_iq3_xxs_f32_impl( gas += args.nb01/2; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2190,6 +2250,23 @@ void kernel_mul_mv_iq3_xxs_f32_impl( } } +template<typename args_t> +void kernel_mul_mv_iq3_xxs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq3_xxs_f32_impl<N_R0_IQ3_XXS_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq3_xxs_f32_impl<N_R0_IQ3_XXS, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq3_xxs_f32")]] kernel void kernel_mul_mv_iq3_xxs_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2201,7 +2278,7 @@ kernel void kernel_mul_mv_iq3_xxs_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq3_xxs_f32_impl<N_R0_IQ3_XXS, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq3_xxs_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template<int nr0, typename args_t> @@ -2227,8 +2304,19 @@ void kernel_mul_mv_iq3_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq3_s * x = (device const block_iq3_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2236,8 +2324,6 @@ void kernel_mul_mv_iq3_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint32_t * svalues = (threadgroup uint32_t *) shmem; { int nval = 8; @@ -2246,11 +2332,9 @@ void kernel_mul_mv_iq3_s_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2265,7 +2349,7 @@ void kernel_mul_mv_iq3_s_f32_impl( device const uint8_t * signs = xr->signs + 4 * ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf)); @@ -2289,7 +2373,7 @@ void kernel_mul_mv_iq3_s_f32_impl( signs += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2302,6 +2386,23 @@ void kernel_mul_mv_iq3_s_f32_impl( } } +template<typename args_t> +void kernel_mul_mv_iq3_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq3_s_f32_impl<N_R0_IQ3_S_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq3_s_f32_impl<N_R0_IQ3_S, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq3_s_f32")]] kernel void kernel_mul_mv_iq3_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2313,7 +2414,7 @@ kernel void kernel_mul_mv_iq3_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq3_s_f32_impl<N_R0_IQ3_S, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq3_s_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template<int nr0, typename args_t> @@ -2339,8 +2440,19 @@ void kernel_mul_mv_iq2_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_s * x = (device const block_iq2_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2348,8 +2460,6 @@ void kernel_mul_mv_iq2_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - //threadgroup uint64_t * svalues = (threadgroup uint64_t *) shmem; //{ // int nval = 32; @@ -2358,11 +2468,9 @@ void kernel_mul_mv_iq2_s_f32_impl( // threadgroup_barrier(mem_flags::mem_threadgroup); //} - const short ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2377,7 +2485,7 @@ void kernel_mul_mv_iq2_s_f32_impl( device const uint8_t * signs = qs + QK_K/8; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const float d1 = db * (0.5f + (sc[0] & 0xf)); const float d2 = db * (0.5f + (sc[0] >> 4)); @@ -2402,7 +2510,7 @@ void kernel_mul_mv_iq2_s_f32_impl( signs += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2415,6 +2523,23 @@ void kernel_mul_mv_iq2_s_f32_impl( } } +template<typename args_t> +void kernel_mul_mv_iq2_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_s_f32_impl<N_R0_IQ2_S_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_s_f32_impl<N_R0_IQ2_S, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_s_f32")]] kernel void kernel_mul_mv_iq2_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2426,7 +2551,7 @@ kernel void kernel_mul_mv_iq2_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_s_f32_impl<N_R0_IQ2_S, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_s_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template<int nr0, typename args_t> @@ -2452,8 +2577,19 @@ void kernel_mul_mv_iq1_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq1_s * x = (device const block_iq1_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2461,13 +2597,9 @@ void kernel_mul_mv_iq1_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - - const short ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { float sumy = 0; for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; @@ -2482,7 +2614,7 @@ void kernel_mul_mv_iq1_s_f32_impl( device const uint16_t * qh = xr->qh + ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700))); constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700))); @@ -2502,7 +2634,7 @@ void kernel_mul_mv_iq1_s_f32_impl( qh += args.nb01/2; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2515,6 +2647,23 @@ void kernel_mul_mv_iq1_s_f32_impl( } } +template<typename args_t> +void kernel_mul_mv_iq1_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq1_s_f32_impl<N_R0_IQ1_S_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq1_s_f32_impl<N_R0_IQ1_S, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq1_s_f32")]] kernel void kernel_mul_mv_iq1_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2525,7 +2674,7 @@ kernel void kernel_mul_mv_iq1_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq1_s_f32_impl<N_R0_IQ1_S, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq1_s_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } template<int nr0, typename args_t> @@ -2551,8 +2700,19 @@ void kernel_mul_mv_iq1_m_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq1_m * x = (device const block_iq1_m *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2560,15 +2720,11 @@ void kernel_mul_mv_iq1_m_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - - const short ix = tiisg; - device const float * y4 = y + 32 * ix; iq1m_scale_t scale; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { float4 sumy = {0.f}; for (short i = 0; i < 8; ++i) { yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0]; @@ -2585,7 +2741,7 @@ void kernel_mul_mv_iq1_m_f32_impl( device const uint8_t * qh = xr->qh + 2 * ib; device const uint16_t * sc = (device const uint16_t *)xr->scales; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); @@ -2611,7 +2767,7 @@ void kernel_mul_mv_iq1_m_f32_impl( qh += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2624,6 +2780,23 @@ void kernel_mul_mv_iq1_m_f32_impl( } } +template<typename args_t> +void kernel_mul_mv_iq1_m_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq1_m_f32_impl<N_R0_IQ1_M_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq1_m_f32_impl<N_R0_IQ1_M, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq1_m_f32")]] kernel void kernel_mul_mv_iq1_m_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2634,7 +2807,7 @@ kernel void kernel_mul_mv_iq1_m_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq1_m_f32_impl<N_R0_IQ1_M, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq1_m_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } template<int NR0, typename args_t> @@ -3213,13 +3386,13 @@ template [[host_name("kernel_mul_mv_id_q3_K_f32")]] kernel kernel_mul_mv_id_t template [[host_name("kernel_mul_mv_id_q4_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q4_K_f32_impl <N_R0_Q4_K>>>; template [[host_name("kernel_mul_mv_id_q5_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q5_K_f32_impl <N_R0_Q5_K>>>; template [[host_name("kernel_mul_mv_id_q6_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q6_K_f32_impl <N_R0_Q6_K>>>; -template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_s_f32_impl <N_R0_IQ1_S>>>; -template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_m_f32_impl <N_R0_IQ1_M>>>; -template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xxs_f32_impl<N_R0_IQ2_XXS>>>; -template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xs_f32_impl <N_R0_IQ2_XS>>>; -template [[host_name("kernel_mul_mv_id_iq3_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq3_xxs_f32_impl<N_R0_IQ3_XXS>>>; -template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq3_s_f32_impl <N_R0_IQ3_S>>>; -template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_s_f32_impl <N_R0_IQ2_S>>>; +template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_s_f32_disp<ggml_metal_kargs_mul_mv>>>; +template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_m_f32_disp<ggml_metal_kargs_mul_mv>>>; +template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xxs_f32_disp<ggml_metal_kargs_mul_mv>>>; +template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xs_f32_disp<ggml_metal_kargs_mul_mv>>>; +template [[host_name("kernel_mul_mv_id_iq3_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq3_xxs_f32_disp<ggml_metal_kargs_mul_mv>>>; +template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq3_s_f32_disp<ggml_metal_kargs_mul_mv>>>; +template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_s_f32_disp<ggml_metal_kargs_mul_mv>>>; template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_nl_f32_impl <N_R0_IQ4_NL>>>; template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_xs_f32_impl <N_R0_IQ4_XS>>>; template [[host_name("kernel_mul_mv_id_tq2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_tq2_0_f32_impl <N_R0_TQ2_0>>>; diff --git a/ggml/src/ggml-metal/kernels/norm.metal b/ggml/src/ggml-metal/kernels/norm.metal index 7e42389fe52d..c76d0c7ff169 100644 --- a/ggml/src/ggml-metal/kernels/norm.metal +++ b/ggml/src/ggml-metal/kernels/norm.metal @@ -1,5 +1,7 @@ #include "common.h" +constant bool FC_norm_use_scale [[function_constant(FC_NORM + 0)]]; + // F == 1 : norm (no fuse) // F == 2 : norm + mul // F == 3 : norm + mul + add @@ -80,7 +82,11 @@ kernel void kernel_norm_fuse_impl( y[i00] = (y[i00]*scale); } if (F == 2) { - y[i00] = (y[i00]*scale)*f0[i00]; + if (FC_norm_use_scale) { + y[i00] = (y[i00]*scale) * args.scale; + } else { + y[i00] = (y[i00]*scale)*f0[i00]; + } } if (F == 3) { y[i00] = (y[i00]*scale)*f0[i00] + f1[i00]; @@ -155,7 +161,11 @@ kernel void kernel_rms_norm_fuse_impl( y[i00] = (x[i00]*scale); } if (F == 2) { - y[i00] = (x[i00]*scale)*f0[i00]; + if (FC_norm_use_scale) { + y[i00] = (x[i00]*scale) * args.scale; + } else { + y[i00] = (x[i00]*scale)*f0[i00]; + } } if (F == 3) { y[i00] = (x[i00]*scale)*f0[i00] + f1[i00]; diff --git a/ggml/src/ggml-metal/kernels/ssm.metal b/ggml/src/ggml-metal/kernels/ssm.metal index d3118a831b95..b21c53b74c7b 100644 --- a/ggml/src/ggml-metal/kernels/ssm.metal +++ b/ggml/src/ggml-metal/kernels/ssm.metal @@ -1,5 +1,8 @@ #include "common.h" +constant bool FC_ssm_conv_silu [[function_constant(FC_SSM_CONV + 1)]]; +constant int FC_ssm_conv_nc [[function_constant(FC_SSM_CONV + 2)]]; + // ref: ggml.c:ggml_compute_forward_ssm_conv_f32 kernel void kernel_ssm_conv_f32_f32( constant ggml_metal_kargs_ssm_conv & args, @@ -13,7 +16,7 @@ kernel void kernel_ssm_conv_f32_f32( const int64_t i2 = tgpig.y; const int64_t i3 = tgpig.z; - const int64_t nc = args.ne10; + const int64_t nc = FC_ssm_conv_nc; //const int64_t ncs = args.ne00; //const int64_t nr = args.ne01; //const int64_t n_t = args.ne1; @@ -25,11 +28,11 @@ kernel void kernel_ssm_conv_f32_f32( float sumf = 0.0f; - for (int64_t i0 = 0; i0 < nc; ++i0) { + FOR_UNROLL (int64_t i0 = 0; i0 < nc; ++i0) { sumf += s[i0] * c[i0]; } - x[0] = sumf; + x[0] = FC_ssm_conv_silu ? sumf/(1.0f + exp(-sumf)) : sumf; } kernel void kernel_ssm_conv_f32_f32_4( @@ -44,7 +47,7 @@ kernel void kernel_ssm_conv_f32_f32_4( const int64_t i2 = tgpig.y; const int64_t i3 = tgpig.z; - const int64_t nc = args.ne10; + const int64_t nc = FC_ssm_conv_nc; //const int64_t ncs = args.ne00; //const int64_t nr = args.ne01; //const int64_t n_t = args.ne1; @@ -56,11 +59,11 @@ kernel void kernel_ssm_conv_f32_f32_4( float sumf = 0.0f; - for (int64_t i0 = 0; i0 < nc/4; ++i0) { + FOR_UNROLL (int64_t i0 = 0; i0 < nc/4; ++i0) { sumf += dot(s[i0], c[i0]); } - x[0] = sumf; + x[0] = FC_ssm_conv_silu ? sumf/(1.0f + exp(-sumf)) : sumf; } constant short FC_ssm_conv_bs [[function_constant(FC_SSM_CONV + 0)]]; @@ -87,7 +90,7 @@ kernel void kernel_ssm_conv_f32_f32_batched( const int64_t i2_off = tpitg.x; const int64_t i2 = i2_base + i2_off; - const int64_t nc = args.ne10; // conv kernel size (typically 4) + const int64_t nc = FC_ssm_conv_nc; // conv kernel size (typically 4) const int64_t n_t = args.ne1; // number of tokens // Bounds check for partial batches at the end @@ -105,11 +108,11 @@ kernel void kernel_ssm_conv_f32_f32_batched( device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2); float sumf = 0.0f; - for (int64_t i0 = 0; i0 < nc; ++i0) { + FOR_UNROLL (int64_t i0 = 0; i0 < nc; ++i0) { sumf += s[i0] * c[i0]; } - x[0] = sumf; + x[0] = FC_ssm_conv_silu ? sumf/(1.0f + exp(-sumf)) : sumf; } kernel void kernel_ssm_conv_f32_f32_batched_4( @@ -132,7 +135,7 @@ kernel void kernel_ssm_conv_f32_f32_batched_4( const int64_t i2_off = tpitg.x; const int64_t i2 = i2_base + i2_off; - const int64_t nc = args.ne10; // conv kernel size (typically 4) + const int64_t nc = FC_ssm_conv_nc; // conv kernel size (typically 4) const int64_t n_t = args.ne1; // number of tokens // Bounds check for partial batches at the end @@ -150,11 +153,11 @@ kernel void kernel_ssm_conv_f32_f32_batched_4( device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2); float sumf = 0.0f; - for (int64_t i0 = 0; i0 < nc/4; ++i0) { + FOR_UNROLL (int64_t i0 = 0; i0 < nc/4; ++i0) { sumf += dot(s[i0], c[i0]); } - x[0] = sumf; + x[0] = FC_ssm_conv_silu ? sumf/(1.0f + exp(-sumf)) : sumf; } // ref: ggml.c:ggml_compute_forward_ssm_scan_f32, Mamba-2 part diff --git a/ggml/src/ggml-musa/CMakeLists.txt b/ggml/src/ggml-musa/CMakeLists.txt index faf9790338bb..82b754f41ee5 100644 --- a/ggml/src/ggml-musa/CMakeLists.txt +++ b/ggml/src/ggml-musa/CMakeLists.txt @@ -43,17 +43,8 @@ if (MUSAToolkit_FOUND) add_compile_definitions(GGML_MUSA_MUDNN_COPY) endif() - if (GGML_CUDA_FA_ALL_QUANTS) - file(GLOB SRCS "../ggml-cuda/template-instances/fattn-vec*.cu") - list(APPEND GGML_SOURCES_MUSA ${SRCS}) - add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS) - else() - list(APPEND GGML_SOURCES_MUSA - ../ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu) - endif() + ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR}/../ggml-cuda SRCS) + list(APPEND GGML_SOURCES_MUSA ${SRCS}) set_source_files_properties(${GGML_SOURCES_MUSA} PROPERTIES LANGUAGE CXX) foreach(SOURCE ${GGML_SOURCES_MUSA}) diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 37e565ef4ff9..ff5e8ef46b70 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -170,6 +170,7 @@ set(GGML_OPENCL_KERNELS gemv_noshuffle_q4_0_f32 gemv_noshuffle_q4_0_f32_spec gemm_noshuffle_q4_0_f32 + gemv_noshuffle_q4_0_f32_32b_trans gemv_noshuffle_q4_1_f32 gemm_noshuffle_q4_1_f32 gemv_noshuffle_q5_0_f32 @@ -184,11 +185,13 @@ set(GGML_OPENCL_KERNELS gemv_noshuffle_q4_k_f32_o4 gemv_noshuffle_q4_k_f32_tiled gemm_noshuffle_q4_k_f32 + gemv_noshuffle_q4_k_f32_32b_trans gemv_noshuffle_q6_k_f32 gemv_noshuffle_q6_k_f32_o4 gemv_noshuffle_q6_k_f32_tiled gemm_noshuffle_q6_k_f32 gemm_noshuffle_q6_k_f32_tiled + gemv_noshuffle_q6_k_f32_32b_trans gemv_noshuffle_q5_k_f32 gemm_noshuffle_q5_k_f32 mul @@ -230,6 +233,7 @@ set(GGML_OPENCL_KERNELS mul_mm_f16_f32_kq_kqv conv2d conv2d_f16_f32 + flash_attn_repack flash_attn_pre_f16 flash_attn_f32_f16 flash_attn_f32_q8_0 diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index ad4a995abf82..239821894049 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -203,39 +203,67 @@ static ggml_cl_version get_opencl_platform_version(cl_platform_id platform) { return parse_cl_version(param_value); } +// Returns the DEVICE's OpenCL version. On an error returns ggml_cl_version with all zeroes. +static ggml_cl_version get_opencl_device_version(cl_device_id device) { + size_t param_size; + if (clGetDeviceInfo(device, CL_DEVICE_VERSION, 0, nullptr, ¶m_size) != CL_SUCCESS || !param_size) { + return {}; + } + std::unique_ptr<char[]> param_storage(new char[param_size]); + if (clGetDeviceInfo(device, CL_DEVICE_VERSION, param_size, param_storage.get(), nullptr) != CL_SUCCESS) { + return {}; + } + + auto param_value = std::string_view(param_storage.get(), param_size); + const std::string version_prefix = "OpenCL "; // "OpenCL <major>.<minor> <device-specific-info>" + if (param_value.find(version_prefix) != 0) { + return {}; + } + param_value.remove_prefix(version_prefix.length()); + return parse_cl_version(param_value); +} + // Return a version to use in OpenCL C compilation. On an error returns ggml_cl_version with all zeroes. static ggml_cl_version get_opencl_c_version(ggml_cl_version platform_version, cl_device_id device) { size_t param_size; #if CL_TARGET_OPENCL_VERSION >= 300 - if (platform_version.major >= 3) { - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, 0, nullptr, ¶m_size)); - if (!param_size) { - return {}; - } + // CL_DEVICE_OPENCL_C_ALL_VERSIONS is an OpenCL 3.0 *device* query, so gating it on the + // *platform* version is not enough: a 3.0 platform can expose 2.0 devices, where the + // query returns CL_INVALID_VALUE and the old CL_CHECK aborted during backend init. + // Gate on the device version, and treat a failure as "fall back to the legacy query" + // rather than fatal -- a device may advertise 3.0 and still refuse the property. + const ggml_cl_version device_version = get_opencl_device_version(device); + if (platform_version.major >= 3 && device_version.major >= 3) { + cl_int err = clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, 0, nullptr, ¶m_size); + if (err == CL_SUCCESS && param_size) { + std::unique_ptr<cl_name_version[]> versions(new cl_name_version[param_size]); + err = clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, param_size, versions.get(), nullptr); + if (err == CL_SUCCESS) { + unsigned versions_count = param_size / sizeof(cl_name_version); - std::unique_ptr<cl_name_version[]> versions(new cl_name_version[param_size]); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, param_size, versions.get(), nullptr)); - unsigned versions_count = param_size / sizeof(cl_name_version); + cl_version version_max = 0; + for (unsigned i = 0; i < versions_count; i++) { + version_max = std::max<cl_version>(versions[i].version, version_max); + } - cl_version version_max = 0; - for (unsigned i = 0; i < versions_count; i++) { - version_max = std::max<cl_version>(versions[i].version, version_max); + return { CL_VERSION_MAJOR(version_max), CL_VERSION_MINOR(version_max) }; + } } - - return { CL_VERSION_MAJOR(version_max), CL_VERSION_MINOR(version_max) }; + // fall through to CL_DEVICE_OPENCL_C_VERSION below } #else GGML_UNUSED(platform_version); #endif // CL_TARGET_OPENCL_VERSION >= 300 - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, 0, nullptr, ¶m_size)); - if (!param_size) { + if (clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, 0, nullptr, ¶m_size) != CL_SUCCESS || !param_size) { return {}; } std::unique_ptr<char[]> param_storage(new char[param_size]); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, param_size, param_storage.get(), nullptr)); + if (clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, param_size, param_storage.get(), nullptr) != CL_SUCCESS) { + return {}; + } auto param_value = std::string_view(param_storage.get(), param_size); const std::string version_prefix = "OpenCL C "; // Suffix: "XX.YY <platform-specific-info>" @@ -539,6 +567,16 @@ struct ggml_opencl_fa_kernels { // attempted (variant, (dk, dv)) // all attempted FA kernels appear here, but those not registered failed compilation std::set<std::pair<int, std::pair<int, int>>> variant_attempted; + + // FA bin kernels +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel_flash_attn_f32_f16_bin; + + cl_kernel kernel_repack_q_for_wmm; + cl_kernel kernel_repack_k_for_wmm; + cl_kernel kernel_repack_v_for_wmm; + cl_kernel kernel_repack_mask_for_wmm; +#endif }; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS @@ -954,6 +992,7 @@ struct ggml_backend_opencl_context { // [size_idx][kda][tgpp] where size_idx: 0=S_V=16, 1=32, 2=64, 3=128; kda: 0 or 1. // tgpp 0 = TG variant (COLS_PER_LANE_GROUP=1), tgpp 1 = prefill variant (COLS_PER_LANE_GROUP=4). cl_kernel kernel_gated_delta_net_f32[4][2][2] = {}; + cl_kernel kernel_ssm_scan_f32 = nullptr; cl_kernel kernel_ssm_scan_f32_mamba2_d128 = nullptr; cl_kernel kernel_ssm_scan_f32_mamba2_d256 = nullptr; @@ -1115,6 +1154,18 @@ struct ggml_backend_opencl_context { } void enqueue_ndrange_kernel(cl_kernel kernel, cl_uint work_dim, size_t *global_work_size, size_t *local_work_size, const ggml_tensor * tensor) { + // From the spec on clEnqueueNDRangeKernel: + // If the device associated with command_queue is an OpenCL 2.1 or newer device, + // and global_work_size is NULL or the value in any passed dimension is zero, + // then the kernel command will trivially succeed after its event dependencies + // are satisfied and subsequently update its completion event. + // So this ensures such cases always return trivially without causing errors in + // case of an older device. + for (cl_uint i = 0; i < work_dim; i++) { + if (global_work_size[i] == 0) { + return; + } + } #ifdef GGML_OPENCL_PROFILING cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, work_dim, NULL, global_work_size, local_work_size, 0, NULL, &evt)); @@ -1160,6 +1211,9 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) + cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q4_0_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; cl_kernel kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; @@ -1183,6 +1237,9 @@ struct ggml_backend_opencl_context { cl_kernel kernel_convert_block_q4_k_tiled_ns; // tiled-wide convert (opt-in) cl_kernel kernel_gemv_noshuffle_q4_k_f32_mc3; // multi-column (N=3) verify GEMV cl_kernel kernel_gemm_noshuffle_q4_k_f32; + cl_kernel kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans; cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in) cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM @@ -1201,6 +1258,9 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemv_noshuffle_q6_K_f32_mc3; // multi-column (N=3) verify GEMV cl_kernel kernel_gemm_noshuffle_q6_K_f32; cl_kernel kernel_gemm_noshuffle_q6_K_f32_cok; + cl_kernel kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q6_k_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q5_k_f32; cl_kernel kernel_gemv_noshuffle_q5_k_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) cl_kernel kernel_gemm_noshuffle_q5_k_f32; @@ -1395,6 +1455,7 @@ static void load_cl_kernels_argsort(ggml_backend_opencl_context *backend_ctx) { static bool use_adreno_bin_kernels(ggml_backend_opencl_context * backend_ctx) { #ifndef GGML_OPENCL_USE_ADRENO_BIN_KERNELS + GGML_UNUSED(backend_ctx); return false; #else if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { @@ -3413,7 +3474,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } - // ssm_scan (Mamba-2 fused per-token recurrent step; d_state in {128, 256}) + // ssm_scan { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { @@ -3425,8 +3486,34 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_ssm_scan_f32 = clCreateKernel(prog, "kernel_ssm_scan_f32", &err), err)); CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d128 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d128", &err), err)); CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d256 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d256", &err), err)); + + cl_kernel * kernels[] = { + &backend_ctx->kernel_ssm_scan_f32_mamba2_d128, + &backend_ctx->kernel_ssm_scan_f32_mamba2_d256 + }; + + // specialized kernels use subgroups and assume subgroup size is 64, + // if device does not support subgroups or subgroup size is not 64, + // release these kernels + for (int i = 0; i < 2; ++i) { + size_t subgroup_size = 0; +#if CL_TARGET_OPENCL_VERSION >= 210 + const size_t local_work_size[] = { 64, 1 }; + const cl_int subgroup_err = clGetKernelSubGroupInfo(*kernels[i], backend_ctx->device, CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE, + sizeof(local_work_size), local_work_size, sizeof(subgroup_size), &subgroup_size, nullptr); + if (subgroup_err != CL_SUCCESS) { + subgroup_size = 0; + } +#endif + // The specialized kernels reduce over one 64-lane subgroup. + if (subgroup_size != 64) { + CL_CHECK(clReleaseKernel(*kernels[i])); + *kernels[i] = nullptr; + } + } CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -3787,6 +3874,55 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = nullptr; + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E) { + { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_0_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_0_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = + clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + } + } + // gemm_noshuffle_q4_1_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4221,6 +4357,104 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = nullptr; + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E) { + { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = + clCreateKernel(prog, "gemv_noshuffle_q4_k_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + } + } + + backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = nullptr; + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E) { + { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q6_k_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = + clCreateKernel(prog, "kernel_gemv_noshuffle_q6_k_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + } + } + std::string CL_moe_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -cl-fast-relaxed-math"; @@ -4987,6 +5221,43 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } + + // repack + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "flash_attn_repack.cl.h" + }; +#else + const std::string kernel_src = read_file("flash_attn_repack.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->fa.kernel_repack_q_for_wmm = clCreateKernel(prog, "kernel_repack_q_for_wmm", &err), err)); + CL_CHECK((backend_ctx->fa.kernel_repack_k_for_wmm = clCreateKernel(prog, "kernel_repack_k_for_wmm", &err), err)); + CL_CHECK((backend_ctx->fa.kernel_repack_v_for_wmm = clCreateKernel(prog, "kernel_repack_v_for_wmm", &err), err)); + CL_CHECK((backend_ctx->fa.kernel_repack_mask_for_wmm = clCreateKernel(prog, "kernel_repack_mask_for_wmm", &err), err)); + GGML_LOG_CONT("."); + } + + // kernel_flash_attn_f32_f16_bin + { + size_t bin_size = 0; + backend_ctx->fa.kernel_flash_attn_f32_f16_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("flash_attn_f32_f16_wmm", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->fa.kernel_flash_attn_f32_f16_bin = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } #endif // GGML_OPENCL_USE_ADRENO_KERNELS GGML_LOG_CONT("\n"); backend_ctx->kernels_loaded = true; @@ -6164,6 +6435,8 @@ static void ggml_opencl_print_backend_info(ggml_backend_opencl_device_context * auto * backend_ctx = dev_ctx->backend_ctx; + GGML_LOG_INFO("ggml_opencl: OpenCL device: %s\n", + backend_ctx->device_name.c_str()); GGML_LOG_INFO("ggml_opencl: OpenCL driver: %s\n", backend_ctx->driver_version.c_str()); GGML_LOG_INFO("ggml_opencl: vector subgroup broadcast support: %s\n", @@ -6180,11 +6453,11 @@ static void ggml_opencl_print_backend_info(ggml_backend_opencl_device_context * backend_ctx->global_mem_size/1024/1024); GGML_LOG_INFO("ggml_opencl: max mem alloc size: %zu MB\n", backend_ctx->max_alloc_size/1024/1024); - GGML_LOG_INFO("ggml_opencl: device max image buffer size (pixels): %lu\n", + GGML_LOG_INFO("ggml_opencl: device max image buffer size (pixels): %zu\n", backend_ctx->image_max_buffer_size); - GGML_LOG_INFO("ggml_opencl: device max image2d size: %lu x %lu\n", + GGML_LOG_INFO("ggml_opencl: device max image2d size: %zu x %zu\n", backend_ctx->image2d_max_width, backend_ctx->image2d_max_height); - GGML_LOG_INFO("ggml_opencl: device max workgroup size: %lu\n", + GGML_LOG_INFO("ggml_opencl: device max workgroup size: %zu\n", backend_ctx->max_workgroup_size); GGML_LOG_INFO("ggml_opencl: SVM coarse grain buffer support: %s\n", backend_ctx->svm_caps & CL_DEVICE_SVM_COARSE_GRAIN_BUFFER ? "true" : "false"); @@ -6725,11 +6998,10 @@ struct ggml_tensor_extra_cl_q4_0 { CL_CHECK(clReleaseMemObject(q_img)); q_img = nullptr; } - // Currently, q_img and d_img are only initialized when SMALL_ALLOC is - // enabled. They point to the images in ggml_backend_opencl_buffer_context. - // So, there is no need to release them here. - // TODO: initialize them for non SMALL_PATH path, or remove them. - d_img = nullptr; + if (d_img != nullptr) { + CL_CHECK(clReleaseMemObject(d_img)); + d_img = nullptr; + } size_q = 0; size_d = 0; } @@ -7147,6 +7419,8 @@ struct ggml_tensor_extra_cl_q6_K { cl_mem ql_img = nullptr; // Upper 2 bits of quantized weights. cl_mem qh = nullptr; + // Upper 2 bits as image1d_buffer_t + cl_mem qh_img = nullptr; // Scales for each block. cl_mem s = nullptr; // Scales for each super block. @@ -7182,6 +7456,10 @@ struct ggml_tensor_extra_cl_q6_K { CL_CHECK(clReleaseMemObject(ql_img)); ql_img = nullptr; } + if (qh_img != nullptr) { + CL_CHECK(clReleaseMemObject(qh_img)); + qh_img = nullptr; + } size_ql = 0; size_qh = 0; @@ -7483,7 +7761,7 @@ static void ggml_cl_moe_bias_glu_fused(ggml_backend_t backend, ggml_tensor * gat size_t global_work_size[] = { (size_t)glu->ne[1]*nth, (size_t)glu->ne[2], 1 }; size_t local_work_size[] = { (size_t)nth, 1, 1 }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, (ggml_tensor *)glu); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, glu); } // Fusion B: the MoE down-projection bias add feeding the combine. @@ -7627,7 +7905,7 @@ static void ggml_cl_moe_bias_combine_fused(ggml_backend_t backend, const ggml_te size_t lws[2] = { 64, 1 }; size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, (ggml_tensor *)dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst); } @@ -7684,6 +7962,7 @@ static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor } inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below (used by the GLU-subgraph fuse check) +inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) { @@ -7738,6 +8017,10 @@ static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx if (use_q4k_tiled(backend_ctx, gate->src[0]) || use_q4k_tiled(backend_ctx, up->src[0])) { return false; } + // q4_K bin kernel requires 32b transposed layout, not compatible with the fused gemv + if (use_q4_k_bin_kernels(backend_ctx, gate->src[0]) || use_q4_k_bin_kernels(backend_ctx, up->src[0])) { + return false; + } // that noshuffle layout is only produced at set_tensor time when // use_adreno_kernels() accepts the weight (ne0 >= 512 && ne1 >= 512). // Smaller weights stay in the plain q4_K layout, which this kernel would @@ -7965,7 +8248,6 @@ static void ggml_cl_mul_mat_q4_k_glu_fused(ggml_backend_t backend, ggml_tensor * GGML_UNUSED(gate_tensor); GGML_UNUSED(up_tensor); GGML_UNUSED(glu_tensor); - GGML_ABORT("q4_K GLU fusion requires GGML_OPENCL_USE_ADRENO_KERNELS"); #endif } @@ -8180,9 +8462,20 @@ inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, c bool threashold_ok = tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 && tensor->ne[2] == 1 && tensor->ne[3] == 1; - // q6_K adreno kernels requires ne1 is multiple of 128 - if (tensor->type == GGML_TYPE_Q6_K) { - return threashold_ok && tensor->ne[1] % 128 == 0; + // The noshuffle layout packs 2 rows per 32-bit texel and the GEMV reads it at an + // ne1/2 texel stride with an exact-cover dispatch, so it is only addressable when + // ne1 is a multiple of 64; an unaligned ne1 truncates the stride and the weight is + // read misaligned. That is a property of the layout, not of one quant -- q4_K, q5_K + // and q8_0 read the same packing as q6_K. The bound is 64, not 128: a q8_0 attention + // weight of ne1 = 2880 is a multiple of 64 but not 128 and is correct. + switch (tensor->type) { + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q8_0: + return threashold_ok && tensor->ne[1] % 64 == 0; + default: + break; } return threashold_ok; } @@ -8304,11 +8597,49 @@ inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *b const size_t elem_num = ggml_nelements(tensor); const size_t q_img_width = elem_num / 8; const size_t qh_img_width = elem_num / 16; + const bool shape_ok = tensor->ne[0] % 32 == 0 && tensor->ne[1] % 4 == 0 && + tensor->ne[2] == 1 && tensor->ne[3] == 1; - return q_img_width <= backend_ctx->image_max_buffer_size && + return shape_ok && q_img_width <= backend_ctx->image_max_buffer_size && qh_img_width <= backend_ctx->image_max_buffer_size; } +inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) { + return false; + } + return (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool use_fa_bin_kernels_prefill(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v) { + if (backend_ctx->fa.kernel_flash_attn_f32_f16_bin == nullptr) { + return false; + } + + const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16; + const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; + + const int n_q = q->ne[1]; + const int dk = q->ne[0]; + const int dv = v->ne[0]; + + constexpr bool prefill_only = true; + + return (backend_ctx->gpu_family == GPU_FAMILY::ADRENO && + (is_mixed || is_q8_0) && (dk == dv) + && (dk == 64 || dk == 128 || dk == 256 || dk == 512) + && (!prefill_only || n_q != 1)); +} +#endif + // The flat-GEMV large-m escape is OPT-IN (GGML_OPENCL_FLAT_LARGE_M=1) because it // is SLOWER than the route it replaces, not because it is unsafe. It was first // parked on the theory that it out-of-bounds-writes at vocab-scale shapes; that @@ -8328,6 +8659,10 @@ static inline bool flat_large_m_enabled() { } static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + if (tensor->ne[1] % 4 != 0 && tensor->ne[2] == 1 && tensor->ne[3] == 1) { + return true; + } + if (!flat_large_m_enabled()) { return false; } @@ -8384,6 +8719,36 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_cont && tensor->ne[2] == 1 && tensor->ne[3] == 1; } +inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin) { + return false; + } + return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && + !use_q6k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin) { + return false; + } + return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && + !use_q4k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *)dev->context; ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; @@ -8566,22 +8931,16 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_SSM_CONV: return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); case GGML_OP_SSM_SCAN: { - // Mamba-2 fused per-token scan. Requires src3->ne[0] == 1 (scalar - // A per head); d_state in {128, 256}; all sources f32. Falls back - // to CPU otherwise (incl. Mamba-1 element-wise A). - for (int i = 0; i < 6; ++i) { - if (op->src[i]->type != GGML_TYPE_F32) { + if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || + op->src[1]->type != GGML_TYPE_F32 || op->src[2]->type != GGML_TYPE_F32 || + op->src[3]->type != GGML_TYPE_F32 || op->src[4]->type != GGML_TYPE_F32 || + op->src[5]->type != GGML_TYPE_F32 || op->src[6]->type != GGML_TYPE_I32) { return false; } + + const int64_t d_state = op->src[0]->ne[0]; + return d_state >= 1 && d_state <= 256 && (d_state & (d_state - 1)) == 0; } - if (op->type != GGML_TYPE_F32) { - return false; - } - const int K = ggml_get_op_params_i32(op, 0); - const int d_state = (int) op->src[0]->ne[0]; - const bool is_mamba2 = (op->src[3]->ne[0] == 1); - return is_mamba2 && (d_state == 128 || d_state == 256) && (K == 1); - } case GGML_OP_GATED_DELTA_NET: { // Match the Vulkan backend: only F32 -> F32, S_v in {16, 32, 64, 128}. @@ -8739,6 +9098,11 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_MEAN: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_FLASH_ATTN_EXT: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_fa_bin_kernels_prefill(backend_ctx, op->src[0], op->src[1], op->src[2])) { + return true; + } +#endif // The E17 compilers segfault while building FA kernels, skip E17 for now if (adreno_e17_compiler_quirks(backend_ctx)) { return false; @@ -9342,6 +9706,96 @@ static enum ggml_status ggml_backend_opencl_buffer_init_tensor(ggml_backend_buff return GGML_STATUS_SUCCESS; } +// Allocate a temporary upload buffer of `nbytes` and populate it with `data` +// from host. On Adreno X1-85 the device-only pool intermittently fails to +// allocate at hundreds of MB once model weights fragment the heap (observed +// on Qwen3.5-9B output.weight Q6_K at 834 MB). Three-step retry: +// 1. CL_MEM_READ_WRITE alloc + clEnqueueWriteBuffer (normal fast path). +// 2. clFinish + retry (drains in-flight allocs that may be holding heap; +// mirrors the proven pattern at the FD-split partial buffer alloc). +// 3. CL_MEM_ALLOC_HOST_PTR + map(WRITE_INVALIDATE) + memcpy + unmap — +// different memory pool (host-pinned); true zero-copy on Adreno per +// QCOM guidance. (CL_MEM_USE_HOST_PTR is NOT zero-copy on Adreno: the +// driver triggers an internal copy because arbitrary host pages aren't +// guaranteed mappable/coherent, AND it draws from the same exhausted +// device pool — so it doesn't solve the problem.) +// Returns the ready-to-read buffer (caller must clReleaseMemObject) or NULL +// if all three strategies fail. The buffer is opaque to the caller — it can +// be passed as a kernel argument like any normal cl_mem. +static cl_mem ggml_cl_create_temp_upload_buffer( + cl_context context, cl_command_queue queue, + size_t nbytes, const void * data, + const char * tensor_name_for_log) +{ + cl_int err; + cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + if (err != CL_SUCCESS) { + clFinish(queue); + buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + } + if (err == CL_SUCCESS) { + const cl_int werr = clEnqueueWriteBuffer(queue, buf, CL_TRUE, 0, nbytes, data, 0, NULL, NULL); + if (werr == CL_SUCCESS) { + return buf; + } + clReleaseMemObject(buf); + } + buf = clCreateBuffer(context, + CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_WRITE_ONLY, + nbytes, NULL, &err); + if (err != CL_SUCCESS) { + return NULL; + } + void * mapped = clEnqueueMapBuffer(queue, buf, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, nbytes, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) { + clReleaseMemObject(buf); + return NULL; + } + memcpy(mapped, data, nbytes); + const cl_int uerr = clEnqueueUnmapMemObject(queue, buf, mapped, 0, NULL, NULL); + if (uerr != CL_SUCCESS) { + clReleaseMemObject(buf); + return NULL; + } + if (tensor_name_for_log) { + GGML_LOG_INFO("ggml_opencl: %s (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", + tensor_name_for_log, nbytes / 1024.0 / 1024.0); + } + return buf; +} + +// Allocate a temporary download buffer of `nbytes`. The caller runs a kernel +// that writes into it, then reads it back to host via clEnqueueReadBuffer (or +// equivalent). Mirrors ggml_cl_create_temp_upload_buffer; the host-pinned +// fallback flags are flipped (CL_MEM_WRITE_ONLY | HOST_READ_ONLY) and the +// helper doesn't populate the buffer. +static cl_mem ggml_cl_create_temp_download_buffer( + cl_context context, cl_command_queue queue, + size_t nbytes, const char * tensor_name_for_log) +{ + cl_int err; + cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + if (err != CL_SUCCESS) { + clFinish(queue); + buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + } + if (err == CL_SUCCESS) { + return buf; + } + buf = clCreateBuffer(context, + CL_MEM_WRITE_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_READ_ONLY, + nbytes, NULL, &err); + if (err != CL_SUCCESS) { + return NULL; + } + if (tensor_name_for_log) { + GGML_LOG_INFO("ggml_opencl: %s download (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", + tensor_name_for_log, nbytes / 1024.0 / 1024.0); + } + return buf; +} + static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; @@ -9450,12 +9904,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // We consider the specified offset arg as always, although For weights // the offset arg should be 0 (we do not assert this). @@ -9567,10 +10017,34 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(K % 32 == 0); - // Transpose q as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - // Transpose d as ushort - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + if (use_q4_0_bin_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; + + // transpose quants as 32-bit words (M-first) + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K / 8, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K / 32, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + + wimg_fmt = { CL_R, CL_HALF_FLOAT }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 32; + wimg_desc.buffer = extra->d; + CL_CHECK((extra->d_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose q and d as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS return; @@ -9589,12 +10063,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_m + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -9721,12 +10191,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -9885,12 +10351,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_m + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10038,12 +10500,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_e + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. @@ -10149,12 +10607,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. @@ -10253,12 +10707,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10337,12 +10787,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_dm + size_s + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q4_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10494,8 +10940,25 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(K % 32 == 0); - // Transpose q, d, dm as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + if (use_q4_k_bin_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; + + // transpose quants as 32-bit words (M-first) + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/8, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose q as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + } transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); @@ -10522,9 +10985,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, "Incorrect tensor size"); cl_int err; - cl_mem data_device; - CL_CHECK((data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err), err)); - CL_CHECK(clEnqueueWriteBuffer(queue, data_device, CL_TRUE, 0, ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q5_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10710,9 +11172,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, "Incorrect tensor size"); cl_int err; - cl_mem data_device; - CL_CHECK((data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err), err)); - CL_CHECK(clEnqueueWriteBuffer(queue, data_device, CL_TRUE, 0, ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q6_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10893,18 +11354,39 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, cl_int M = tensor->ne[1]; // ne01 cl_int K = tensor->ne[0]; // ne00 - // Transpose ql as ushort - transpose_2d_as_16b(backend_ctx, - extra->ql, extra->ql, size_ql, K/4, M); - - // Transpose qh as uchar - transpose_2d_as_8b(backend_ctx, - extra->qh, extra->qh, size_qh, K/4, M); + if (use_q6_k_bin_kernels(backend_ctx, tensor)) { + GGML_ASSERT(K % 256 == 0); + GGML_ASSERT(M % 64 == 0); + + transpose_2d_as_32b(backend_ctx, extra->ql, extra->ql, size_ql, K/8, M); + transpose_2d_as_32b(backend_ctx, extra->qh, extra->qh, size_qh, K/16, M); + + cl_image_format wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + cl_image_desc wimg_desc; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = static_cast<size_t>(ggml_nelements(tensor) / 8); + wimg_desc.buffer = extra->ql; + CL_CHECK((extra->ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = static_cast<size_t>(ggml_nelements(tensor) / 16); + wimg_desc.buffer = extra->qh; + CL_CHECK((extra->qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose ql as ushort + transpose_2d_as_16b(backend_ctx, + extra->ql, extra->ql, size_ql, K/4, M); - // Transpose s as ushort - transpose_2d_as_16b(backend_ctx, - extra->s, extra->s, size_s, K/16/2, M); + // Transpose qh as uchar + transpose_2d_as_8b(backend_ctx, + extra->qh, extra->qh, size_qh, K/4, M); + // Transpose s as ushort + transpose_2d_as_16b(backend_ctx, + extra->s, extra->s, size_s, K/16/2, M); + } // Transpose d as ushort transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); @@ -11050,12 +11532,10 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_trans4_ns; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; @@ -11098,7 +11578,11 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + if (use_q4_0_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 8); + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 4); + } transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); cl_uchar mask_0F = 0x0F; @@ -11120,10 +11604,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, } #endif - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); @@ -11148,10 +11630,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_trans4_ns; int ne00 = tensor->ne[0]; @@ -11223,10 +11703,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, } #endif - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); @@ -11252,11 +11730,9 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; // TODO: use ggml_cl_buffer to manage this temporary buffer - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_trans4_ns; @@ -11356,11 +11832,9 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; // TODO: use ggml_cl_buffer to manage this temporary buffer - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_trans4_ns; @@ -11465,10 +11939,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_MXFP4) { ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { @@ -11530,10 +12002,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; ggml_tensor_extra_cl_q8_0 * extra = (ggml_tensor_extra_cl_q8_0 *)extra_src->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (enable_adreno_trans_weight(backend_ctx, tensor)) { @@ -11586,10 +12056,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_IQ4_NL) { ggml_tensor_extra_cl_iq4_nl * extra = (ggml_tensor_extra_cl_iq4_nl *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_kernels(backend_ctx, tensor)) { @@ -11658,10 +12126,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_Q4_K) { ggml_tensor_extra_cl_q4_K * extra = (ggml_tensor_extra_cl_q4_K *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; @@ -11716,10 +12182,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_k_trans4_ns; @@ -11769,7 +12233,11 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, buf_trans_s.allocate(backend_ctx->context, size_s); // Transpose q, d, dm, s back - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + if (use_q4_k_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8); + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + } transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true); @@ -11820,20 +12288,16 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_Q5_K) { ggml_tensor_extra_cl_q5_K * extra = (ggml_tensor_extra_cl_q5_K *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_k_trans4_ns; int ne00 = tensor->ne[0]; @@ -11993,10 +12457,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q6_k_trans4_ns; @@ -12049,15 +12511,24 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, buf_trans_ql.allocate(backend_ctx->context, size_ql); buf_trans_qh.allocate(backend_ctx->context, size_qh); - buf_trans_s.allocate(backend_ctx->context, size_s); buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - // transpose ql, qh, s and d back - transpose_2d_as_16b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/4); - transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/4); - transpose_2d_as_16b(backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/16/2); - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); + cl_mem s_buffer; + if (use_q6_k_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/8); + transpose_2d_as_32b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/16); + // s is left row-major, untransposed, for the binary layout. + s_buffer = extra->s; + } else { + // transpose ql, qh, s and d back + buf_trans_s.allocate(backend_ctx->context, size_s); + transpose_2d_as_16b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/4); + transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/4); + transpose_2d_as_16b(backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/16/2); + s_buffer = buf_trans_s.buffer; + } + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); // unpack cl_uchar mask = 0xFF; @@ -12065,7 +12536,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_ql.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_s.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s_buffer)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); @@ -12083,10 +12554,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, } #endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask = 0xFF; cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); @@ -12214,6 +12683,21 @@ static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_b cl_int err; cl_mem mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); + // On Adreno X1-85 the device pool intermittently fails at hundreds of MB + // once the heap fragments (e.g. graph-allocator compute-buffer reserve + // after model load). Four-step retry: + // 1. normal alloc (fast path) + // 2. clFinish + retry (drains in-flight allocs) + // 3. cl_qcom_large_buffer (X2-class driver only, OpenCL 3.0 only) + // 4. ALLOC_HOST_PTR (host-pinned pool) — last-resort fallback. This + // buffer backs compute scratch read/written by every kernel in the + // graph, so kernel accesses fall to host memory and runtime perf + // degrades meaningfully. Better than failing to load, but the user + // should see the warning and consider -ngl reduction. + if (err != CL_SUCCESS) { + clFinish(backend_ctx->queue); + mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); + } #if GGML_OPENCL_TARGET_VERSION >= 300 // clCreateBufferWithProperties and cl_mem_properties are OpenCL 3.0. Drivers older than // that do not export the symbol, so a build targeting them fails to link. The large @@ -12224,9 +12708,20 @@ static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_b mem = clCreateBufferWithProperties(backend_ctx->context, props, CL_MEM_READ_WRITE, size, NULL, &err); } #endif + if (err != CL_SUCCESS) { + mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, size, NULL, &err); + if (err == CL_SUCCESS) { + GGML_LOG_WARN("%s: %.2f MiB allocated via CL_MEM_ALLOC_HOST_PTR fallback — " + "device pool exhausted; runtime perf will be degraded. " + "Consider lowering -ngl or context size.\n", + __func__, size / 1024.0 / 1024.0); + } + } if (err != CL_SUCCESS) { - GGML_LOG_INFO("%s: failed to allocate %.2f MiB\n", __func__, size / 1024.0 / 1024.0); + GGML_LOG_ERROR("%s: failed to allocate %.2f MiB (err=%d). " + "Consider reducing -ngl, lowering -c / -ub, or using quantized KV cache.\n", + __func__, size / 1024.0 / 1024.0, err); return nullptr; } @@ -12981,6 +13476,7 @@ static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, c (size_t)ne03}; size_t local_work_size[] = {(size_t)nth0, (size_t)rows_per_workgroup, 1}; + // ne01 == 0 makes global_work_size[0] zero here; enqueue_ndrange_kernel drops the empty range. backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } @@ -13770,81 +14266,109 @@ static void ggml_cl_mean(ggml_backend_t backend, const ggml_tensor * src0, const } static void ggml_cl_ssm_scan(ggml_backend_t backend, ggml_tensor * dst) { - const ggml_tensor * src0 = dst->src[0]; // s - const ggml_tensor * src1 = dst->src[1]; // x - const ggml_tensor * src2 = dst->src[2]; // dt - const ggml_tensor * src3 = dst->src[3]; // A - const ggml_tensor * src4 = dst->src[4]; // B - const ggml_tensor * src5 = dst->src[5]; // C - const ggml_tensor * src6 = dst->src[6]; // ids - - GGML_ASSERT(src0 && src1 && src2 && src3 && src4 && src5 && src6 && dst); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_ASSERT(dst->src[0]); + GGML_ASSERT(dst->src[0]->extra); + GGML_ASSERT(dst->src[1]); + GGML_ASSERT(dst->src[1]->extra); + GGML_ASSERT(dst->src[2]); + GGML_ASSERT(dst->src[2]->extra); + GGML_ASSERT(dst->src[3]); + GGML_ASSERT(dst->src[3]->extra); + GGML_ASSERT(dst->src[4]); + GGML_ASSERT(dst->src[4]->extra); + GGML_ASSERT(dst->src[5]); + GGML_ASSERT(dst->src[5]->extra); + GGML_ASSERT(dst->src[6]); + GGML_ASSERT(dst->src[6]->extra); ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; - ggml_tensor_extra_cl * e0 = (ggml_tensor_extra_cl *) src0->extra; - ggml_tensor_extra_cl * e1 = (ggml_tensor_extra_cl *) src1->extra; - ggml_tensor_extra_cl * e2 = (ggml_tensor_extra_cl *) src2->extra; - ggml_tensor_extra_cl * e3 = (ggml_tensor_extra_cl *) src3->extra; - ggml_tensor_extra_cl * e4 = (ggml_tensor_extra_cl *) src4->extra; - ggml_tensor_extra_cl * e5 = (ggml_tensor_extra_cl *) src5->extra; - ggml_tensor_extra_cl * e6 = (ggml_tensor_extra_cl *) src6->extra; - ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *) dst->extra; - - cl_ulong o0 = e0->offset + src0->view_offs; - cl_ulong o1 = e1->offset + src1->view_offs; - cl_ulong o2 = e2->offset + src2->view_offs; - cl_ulong o3 = e3->offset + src3->view_offs; - cl_ulong o4 = e4->offset + src4->view_offs; - cl_ulong o5 = e5->offset + src5->view_offs; - cl_ulong o6 = e6->offset + src6->view_offs; - cl_ulong od = ed->offset + dst->view_offs; - - const int d_state = (int) src0->ne[0]; - const int head_dim = (int) src0->ne[1]; - const int n_head = (int) src1->ne[1]; - const int n_group = (int) src4->ne[1]; - const int n_tokens = (int) src1->ne[2]; - const int n_seqs = (int) src1->ne[3]; - - // Mirror CPU ref: s_off = ggml_nelements(src1) * sizeof(float) - const cl_ulong s_off_bytes = (cl_ulong) ggml_nelements(src1) * sizeof(float); - - cl_kernel kernel = (d_state == 128) - ? backend_ctx->kernel_ssm_scan_f32_mamba2_d128 - : backend_ctx->kernel_ssm_scan_f32_mamba2_d256; - GGML_ASSERT(kernel != nullptr); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) dst->src[0]->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) dst->src[1]->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *) dst->src[2]->extra; + ggml_tensor_extra_cl * extra3 = (ggml_tensor_extra_cl *) dst->src[3]->extra; + ggml_tensor_extra_cl * extra4 = (ggml_tensor_extra_cl *) dst->src[4]->extra; + ggml_tensor_extra_cl * extra5 = (ggml_tensor_extra_cl *) dst->src[5]->extra; + ggml_tensor_extra_cl * extra6 = (ggml_tensor_extra_cl *) dst->src[6]->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong offset0 = extra0->offset + dst->src[0]->view_offs; + const cl_ulong offset1 = extra1->offset + dst->src[1]->view_offs; + const cl_ulong offset2 = extra2->offset + dst->src[2]->view_offs; + const cl_ulong offset3 = extra3->offset + dst->src[3]->view_offs; + const cl_ulong offset4 = extra4->offset + dst->src[4]->view_offs; + const cl_ulong offset5 = extra5->offset + dst->src[5]->view_offs; + const cl_ulong offset6 = extra6->offset + dst->src[6]->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; + + const ggml_tensor * s = dst->src[0]; + const ggml_tensor * x = dst->src[1]; + const ggml_tensor * dt = dst->src[2]; + const ggml_tensor * A = dst->src[3]; + const ggml_tensor * B = dst->src[4]; + const ggml_tensor * C = dst->src[5]; + + const cl_ulong s_nb1 = s->nb[1]; + const cl_ulong s_nb2 = s->nb[2]; + const cl_ulong s_nb3 = s->nb[3]; + const cl_ulong x_nb1 = x->nb[1]; + const cl_ulong x_nb2 = x->nb[2]; + const cl_ulong x_nb3 = x->nb[3]; + const cl_ulong dt_nb1 = dt->nb[1]; + const cl_ulong dt_nb2 = dt->nb[2]; + const cl_ulong A_nb1 = A->nb[1]; + const cl_ulong B_nb1 = B->nb[1]; + const cl_ulong B_nb2 = B->nb[2]; + const cl_ulong B_nb3 = B->nb[3]; + const cl_ulong C_nb1 = C->nb[1]; + const cl_ulong C_nb2 = C->nb[2]; + const cl_ulong C_nb3 = C->nb[3]; + + const cl_uint A_ne0 = A->ne[0]; + const cl_uint d_state = s->ne[0]; + const cl_int head_dim = x->ne[0]; + const cl_int n_head = x->ne[1]; + const cl_int n_group = B->ne[1]; + const cl_int n_tokens = x->ne[2]; + const cl_uint n_seqs = x->ne[3]; + const cl_uint K = ggml_get_op_params_i32(dst, 0); + const cl_ulong s_off_bytes = (cl_ulong) ggml_nelements(x) * sizeof(float); + + cl_kernel kernel = backend_ctx->kernel_ssm_scan_f32; + size_t nth = d_state; + if (A_ne0 == 1 && K == 1) { + cl_kernel kernel_mamba2 = nullptr; + if (d_state == 128) { + kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d128; + } else if (d_state == 256) { + kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d256; + } + if (kernel_mamba2 != nullptr) { + kernel = kernel_mamba2; + nth = 64; + } + } - cl_ulong s0_nb2 = src0->nb[2]; - cl_ulong s0_nb3 = src0->nb[3]; - cl_ulong x_nb2 = src1->nb[2]; - cl_ulong x_nb3 = src1->nb[3]; - cl_ulong dt_nb1 = src2->nb[1]; - cl_ulong dt_nb2 = src2->nb[2]; - cl_ulong A_nb1 = src3->nb[1]; - cl_ulong B_nb2 = src4->nb[2]; - cl_ulong B_nb3 = src4->nb[3]; - cl_ulong C_nb2 = src5->nb[2]; - cl_ulong C_nb3 = src5->nb[3]; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &e0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &o0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &e1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &o1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &e2->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &o2)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &e3->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &o3)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &e4->data_device)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &o4)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), &e5->data_device)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &o5)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), &e6->data_device)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &o6)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_mem), &ed->data_device)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &od)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &s0_nb2)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &s0_nb3)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra3->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extra4->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offset4)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), &extra5->data_device)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &offset5)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), &extra6->data_device)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &offset6)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &s_nb2)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &s_nb3)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &x_nb2)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &x_nb3)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &dt_nb1)); @@ -13855,15 +14379,30 @@ static void ggml_cl_ssm_scan(ggml_backend_t backend, ggml_tensor * dst) { CL_CHECK(clSetKernelArg(kernel, 25, sizeof(cl_ulong), &C_nb2)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &C_nb3)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &s_off_bytes)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(int), &head_dim)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &n_group)); - CL_CHECK(clSetKernelArg(kernel, 31, sizeof(int), &n_tokens)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_int), &head_dim)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_int), &n_group)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_int), &n_tokens)); + + if (kernel == backend_ctx->kernel_ssm_scan_f32) { + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &s_nb1)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &x_nb1)); + CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &B_nb1)); + CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &C_nb1)); + CL_CHECK(clSetKernelArg(kernel, 36, sizeof(cl_uint), &A_ne0)); + CL_CHECK(clSetKernelArg(kernel, 37, sizeof(cl_uint), &d_state)); + CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_uint), &n_seqs)); + CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_uint), &K)); + CL_CHECK(clSetKernelArg(kernel, 40, d_state * sizeof(float), nullptr)); + } - size_t global_work_size[] = { (size_t)n_head * head_dim * 64, (size_t)n_seqs, 1 }; - size_t local_work_size[] = { 64, 1, 1 }; + size_t global_work_size[] = { + (size_t) head_dim * (size_t) n_head * nth, + (size_t) n_seqs, + }; + size_t local_work_size[] = { nth, 1 }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } static void ggml_cl_ssm_conv(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -16772,6 +17311,407 @@ static void ggml_cl_adreno_xmem_attn_run( #endif // GGML_OPENCL_USE_ADRENO_KERNELS +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + GGML_ASSERT(q->extra); + GGML_ASSERT(k->extra); + GGML_ASSERT(v->extra); + GGML_ASSERT(dst->extra); + if (mask) { + GGML_ASSERT(mask->extra); + } + if (sinks) { + GGML_ASSERT(sinks->extra); + } + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_context context = backend_ctx->context; + + const int n_q = q->ne[1]; + const int n_kv = k->ne[1]; + const int d_head_q = q->ne[0]; + const int d_head_v = v->ne[0]; + const int n_head = q->ne[2]; + const int n_head_kv = k->ne[2]; + const int n_batch = q->ne[3]; + + const std::pair<int, int> dk_dv = {d_head_q, d_head_v}; + cl_kernel kernel = backend_ctx->fa.kernel_flash_attn_f32_f16_bin; + GGML_ASSERT(kernel != NULL); + + ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra; + ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *)k->extra; + ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra; + ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL; + ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL; + + cl_ulong offset_q = extra_q->offset + q->view_offs; + cl_ulong offset_o = extra_o->offset + dst->view_offs; + + cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL; + cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; + cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL; + cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0; + + const cl_ulong q_nb1 = q->nb[1]; + const cl_ulong q_nb2 = q->nb[2]; + const cl_ulong q_nb3 = q->nb[3]; + + cl_mem k_data_device = extra_k->data_device; + cl_ulong offset_k = extra_k->offset + k->view_offs; + cl_ulong k_nb1 = k->nb[1]; + cl_ulong k_nb2 = k->nb[2]; + cl_ulong k_nb3 = k->nb[3]; + + cl_mem v_data_device = extra_v->data_device; + cl_ulong offset_v = extra_v->offset + v->view_offs; + cl_ulong v_nb1 = v->nb[1]; + cl_ulong v_nb2 = v->nb[2]; + cl_ulong v_nb3 = v->nb[3]; + + const cl_ulong o_nb1 = dst->nb[1]; + const cl_ulong o_nb2 = dst->nb[2]; + const cl_ulong o_nb3 = dst->nb[3]; + + const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0; + const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0; + const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0; + const int mask_ne2 = mask ? mask->ne[2] : 0; + const int mask_ne3 = mask ? mask->ne[3] : 0; + + float * params = (float *)dst->op_params; + float scale = params[0]; + float max_bias = params[1]; + float logit_softcap = params[2]; + + const int is_causal = (mask == NULL && n_q > 1 && n_q == n_kv); // redundant n_q > 1 check ? + + const int n_head_log2_val = n_head > 0 ? 1u << (int)floorf(log2f((float)n_head)) : 0; + const float n_head_log2_f = n_head_log2_val > 0 ? (float)n_head_log2_val : 1.0f; + const float m0 = powf(2.0f, -(max_bias) / n_head_log2_f); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2_f); + + const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; + + ggml_cl_flash_attn_temp_buffer temp_k; + ggml_cl_flash_attn_temp_buffer temp_v; + ggml_cl_flash_attn_temp_buffer temp_k_aos; + ggml_cl_flash_attn_temp_buffer temp_v_aos; + + if (is_q8_0) { + ggml_cl_flash_attn_reconstruct_aos( + backend_ctx, k, temp_k_aos, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + + ggml_cl_flash_attn_reconstruct_aos( + backend_ctx, v, temp_v_aos, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + + bool k_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, k, GGML_TYPE_F16, k_data_device, offset_k, k_nb1, k_nb2, k_nb3, + temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + + bool v_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, v, GGML_TYPE_F16, v_data_device, offset_v, v_nb1, v_nb2, v_nb3, + temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + + GGML_ASSERT(k_done && v_done); + } + + // Allocate input/output memory buffers + cl_mem mem_matrixQ; + cl_mem mem_matrixK; + cl_mem mem_matrixV; + cl_mem mem_matrixO; + cl_buffer_region region; + cl_int err; + + region.origin = offset_q; + region.size = ggml_nbytes(q); + mem_matrixQ = clCreateSubBuffer(extra_q->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + region.origin = offset_k; + region.size = is_q8_0 ? (size_t) k_nb3 * (size_t) k->ne[3] : ggml_nbytes(k); + mem_matrixK = clCreateSubBuffer(k_data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + region.origin = offset_v; + region.size = is_q8_0 ? (size_t) v_nb3 * (size_t) v->ne[3] : ggml_nbytes(v); + mem_matrixV = clCreateSubBuffer(v_data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + region.origin = offset_o; + region.size = ggml_nbytes(dst); + mem_matrixO = clCreateSubBuffer(extra_o->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt_1d = { CL_RGBA, CL_FLOAT}; + cl_image_desc img_desc_1d; + + // use image 1d buffer used as fallback when on mask is applied + cl_mem mem_tex_mask_fallback_1dbuf; + img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc_1d, 0, sizeof(img_desc_1d)); + img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_1d.image_width = 1; + img_desc_1d.buffer = mem_matrixK; + mem_tex_mask_fallback_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &err); + CL_CHECK(err); + + cl_mem mem_tex_matrixO_1dbuf; + img_fmt_1d = { CL_RGBA, CL_FLOAT}; + memset(&img_desc_1d, 0, sizeof(img_desc_1d)); + img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_1d.image_width = ggml_nbytes(dst) / 4 / 4; + img_desc_1d.buffer = mem_matrixO; + mem_tex_matrixO_1dbuf = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &err); + CL_CHECK(err); + + // The bin kernel requires 2d (or 3d) buffers packed for data loading/multiplication. + // These repack kernels launch across all buffers to ensure compatibility + cl_mem mem_tex_matrixMask_1dbuf = NULL; + cl_mem mem_matrixMask = NULL; + cl_mem mem_matrixMask_padded = NULL; + cl_ulong mask_nb1_padded = mask_nb1, mask_nb2_padded = mask_nb2, mask_nb3_padded = mask_nb3; + if (extra_mask) { + // allocate mem_matrixMask w/ new padded size + size_t n_kv_padded = GGML_PAD(n_kv, 4); + size_t mask_nb_padded = n_kv_padded * sizeof(cl_half) * mask->ne[1] * mask->ne[2] * mask->ne[3]; + + // apply offset and create subBuffer for mask + region.origin = offset_mask; + region.size = ggml_nbytes(mask); + mem_matrixMask = clCreateSubBuffer(extra_mask->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + { + // create padded mask to contain all data + mem_matrixMask_padded = clCreateBuffer(context, CL_MEM_ALLOC_HOST_PTR, mask_nb_padded, NULL, &err); + CL_CHECK(err); + + // pass extra_mask->data_device, mem_matrixMask to kernel for copying/padding + mask_nb1_padded = (cl_ulong)n_kv_padded * sizeof(cl_half); + mask_nb2_padded = mask_nb1_padded * (cl_ulong)mask->ne[1]; + mask_nb3_padded = mask_nb2_padded * (cl_ulong)mask->ne[2]; + + cl_kernel repack_mask = backend_ctx->fa.kernel_repack_mask_for_wmm; + CL_CHECK(clSetKernelArg(repack_mask, 0, sizeof(cl_mem), &mem_matrixMask)); + CL_CHECK(clSetKernelArg(repack_mask, 1, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(repack_mask, 2, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(repack_mask, 3, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(repack_mask, 4, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(repack_mask, 5, sizeof(cl_mem), &mem_matrixMask_padded)); + CL_CHECK(clSetKernelArg(repack_mask, 6, sizeof(cl_ulong), &mask_nb1_padded)); + CL_CHECK(clSetKernelArg(repack_mask, 7, sizeof(cl_ulong), &mask_nb2_padded)); + CL_CHECK(clSetKernelArg(repack_mask, 8, sizeof(cl_ulong), &mask_nb3_padded)); + + size_t repack_mask_gws[3] = {(size_t)n_kv, (size_t)mask->ne[1], (size_t)mask_ne2 * (size_t)mask->ne[3]}; + backend_ctx->enqueue_ndrange_kernel(repack_mask, 3, repack_mask_gws, NULL, dst); + } + + // use image 1d buffer for matrix Mask (padded row stride) + cl_image_format img_fmt_mask_1d = { CL_RGBA, CL_HALF_FLOAT}; + cl_image_desc img_desc_mask_1d; + memset(&img_desc_mask_1d, 0, sizeof(img_desc_mask_1d)); + img_desc_mask_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_mask_1d.image_width = mask_nb_padded / 2 / 4; + img_desc_mask_1d.buffer = mem_matrixMask_padded; + mem_tex_matrixMask_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_mask_1d, &img_desc_mask_1d, NULL, &err); + CL_CHECK(err); + } + + // WMM QK uses repacked 3D images. + // Q image: rows, heads, packed depth. + cl_image_format img_fmt_3d = { CL_RGBA, CL_HALF_FLOAT }; + cl_image_desc img_desc_3d; + + memset(&img_desc_3d, 0, sizeof(img_desc_3d)); + img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; + img_desc_3d.image_width = (size_t)n_q; + img_desc_3d.image_height = (size_t)n_batch * (size_t)n_head; + img_desc_3d.image_depth = (size_t)d_head_q / 4; + cl_mem img_q_wmm = NULL; + img_q_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); + CL_CHECK(err); + + { + cl_kernel repack_q = backend_ctx->fa.kernel_repack_q_for_wmm; + CL_CHECK(clSetKernelArg(repack_q, 0, sizeof(cl_mem), &mem_matrixQ)); + CL_CHECK(clSetKernelArg(repack_q, 1, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(repack_q, 2, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(repack_q, 3, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(repack_q, 4, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(repack_q, 5, sizeof(cl_mem), &img_q_wmm)); + + size_t repack_q_gws[3] = {(size_t)d_head_q / 4, (size_t)n_q, (size_t)n_batch * (size_t)n_head}; + backend_ctx->enqueue_ndrange_kernel(repack_q, 3, repack_q_gws, NULL, dst); + } + + // K image: columns, row groups, KV heads. + const size_t n_kv_row4 = ((size_t)n_kv + 3) / 4; + + memset(&img_desc_3d, 0, sizeof(img_desc_3d)); + img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; + img_desc_3d.image_width = (size_t)d_head_q; + img_desc_3d.image_height = n_kv_row4; + img_desc_3d.image_depth = (size_t)n_batch * (size_t)n_head_kv; + cl_mem img_k_wmm = NULL; + img_k_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); + CL_CHECK(err); + + { + cl_kernel repack_k = backend_ctx->fa.kernel_repack_k_for_wmm; + CL_CHECK(clSetKernelArg(repack_k, 0, sizeof(cl_mem), &mem_matrixK)); + CL_CHECK(clSetKernelArg(repack_k, 1, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(repack_k, 2, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(repack_k, 3, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(repack_k, 4, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(repack_k, 5, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(repack_k, 6, sizeof(cl_mem), &img_k_wmm)); + + size_t repack_k_gws[3] = {(size_t)d_head_q, n_kv_row4, (size_t)n_batch * (size_t)n_head_kv}; + backend_ctx->enqueue_ndrange_kernel(repack_k, 3, repack_k_gws, NULL, dst); + } + + // V image: kv-rows (contracted), packed head-dim groups, KV heads. + memset(&img_desc_3d, 0, sizeof(img_desc_3d)); + img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; + img_desc_3d.image_width = (size_t)n_kv; + img_desc_3d.image_height = (size_t)d_head_v / 4; + img_desc_3d.image_depth = (size_t)n_batch * (size_t)n_head_kv; + cl_mem img_v_wmm = NULL; + img_v_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); + CL_CHECK(err); + + { + cl_kernel repack_v = backend_ctx->fa.kernel_repack_v_for_wmm; + CL_CHECK(clSetKernelArg(repack_v, 0, sizeof(cl_mem), &mem_matrixV)); + CL_CHECK(clSetKernelArg(repack_v, 1, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(repack_v, 2, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(repack_v, 3, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(repack_v, 4, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(repack_v, 5, sizeof(cl_mem), &img_v_wmm)); + + size_t repack_v_gws[3] = {(size_t)d_head_v / 4, (size_t)n_kv, (size_t)n_batch * (size_t)n_head_kv}; + backend_ctx->enqueue_ndrange_kernel(repack_v, 3, repack_v_gws, NULL, dst); + } + + cl_int enable_mask = (extra_mask) ? 1 : 0; + mask_buffer = extra_mask ? mem_tex_matrixMask_1dbuf : mem_tex_mask_fallback_1dbuf; + + cl_mem mem_sinksBuf = NULL; + cl_mem mem_tex_sinks_1dbuf = NULL; + cl_int enable_sinks = (sinks_buffer != NULL) ? 1 : 0; + if (enable_sinks) { + region.origin = offset_sinks; + region.size = ggml_nbytes(sinks); + mem_sinksBuf = clCreateSubBuffer(extra_sinks->data_device, CL_MEM_READ_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt_sinks_1d = { CL_R, CL_FLOAT }; + cl_image_desc img_desc_sinks_1d; + memset(&img_desc_sinks_1d, 0, sizeof(img_desc_sinks_1d)); + img_desc_sinks_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_sinks_1d.image_width = (size_t)n_head; + img_desc_sinks_1d.buffer = mem_sinksBuf; + mem_tex_sinks_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_sinks_1d, &img_desc_sinks_1d, NULL, &err); + CL_CHECK(err); + } else { + // The image obj cannot be null so we back with buffer of size 1 and use matrixK to back because it always exists + cl_image_format img_fmt_sinks_fallback = { CL_R, CL_FLOAT }; + cl_image_desc img_desc_sinks_fallback; + memset(&img_desc_sinks_fallback, 0, sizeof(img_desc_sinks_fallback)); + img_desc_sinks_fallback.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_sinks_fallback.image_width = 1; + img_desc_sinks_fallback.buffer = mem_matrixK; + mem_tex_sinks_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_sinks_fallback, &img_desc_sinks_fallback, NULL, &err); + CL_CHECK(err); + } + + cl_uint arg = 0; + + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mem_tex_matrixO_1dbuf)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &is_causal)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head_log2_val)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &logit_softcap)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &enable_mask)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb1_padded)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb2_padded)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb3_padded)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &mask_ne3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mem_tex_sinks_1dbuf)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &enable_sinks)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_q_wmm)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_k_wmm)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_v_wmm)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &d_head_q)); + + size_t global_work_size[3], local_work_size[3]; + + const int n_waves_v = d_head_q / 64; + + local_work_size[0] = 64; + local_work_size[1] = n_waves_v; + local_work_size[2] = 1; + + global_work_size[0] = 64; + global_work_size[1] = ((n_q + 64 - 1) / 64) * n_waves_v; + global_work_size[2] = n_batch * n_head; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(mem_tex_matrixO_1dbuf)); + CL_CHECK(clReleaseMemObject(img_q_wmm)); + CL_CHECK(clReleaseMemObject(img_k_wmm)); + CL_CHECK(clReleaseMemObject(img_v_wmm)); + + if (mem_tex_matrixMask_1dbuf) { + CL_CHECK(clReleaseMemObject(mem_tex_matrixMask_1dbuf)); + } + if (mem_matrixMask) { + CL_CHECK(clReleaseMemObject(mem_matrixMask)); + } + if (mem_matrixMask_padded) { + CL_CHECK(clReleaseMemObject(mem_matrixMask_padded)); + } + if (mem_tex_sinks_1dbuf) { + CL_CHECK(clReleaseMemObject(mem_tex_sinks_1dbuf)); + } + if (mem_sinksBuf) { + CL_CHECK(clReleaseMemObject(mem_sinksBuf)); + } + CL_CHECK(clReleaseMemObject(mem_matrixQ)); + CL_CHECK(clReleaseMemObject(mem_matrixK)); + CL_CHECK(clReleaseMemObject(mem_matrixV)); + CL_CHECK(clReleaseMemObject(mem_matrixO)); +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { const ggml_tensor * v = dst->src[2]; const ggml_tensor * mask = dst->src[3]; @@ -16827,6 +17767,14 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; const bool is_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 && v->type == GGML_TYPE_Q4_0; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_fa_bin_kernels_prefill(backend_ctx, q, k, v)) { + // We support the prefill path of flash attn with a specialized d_head = 64/128/256 + ggml_cl_flash_attn_prefill_bin(backend, q, k, dst); + return; + } +#endif + if (is_f16) { ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F16); } else if (is_mixed) { @@ -17900,16 +18848,34 @@ static void ggml_cl_conv_2d(ggml_backend_t backend, const ggml_tensor * src0, co cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - const cl_uint Cout = ne03; const cl_uint Cin = ne02; const cl_uint N = ne13; - const cl_uint KW = ne00; const cl_uint KH = ne01; const cl_uint W = ne10; const cl_uint H = ne11; const cl_uint OW = ne0; const cl_uint OH = ne1; - - const cl_uint s0 = dst->op_params[0]; const cl_uint s1 = dst->op_params[1]; - const cl_uint p0 = dst->op_params[2]; const cl_uint p1 = dst->op_params[3]; - const cl_uint d0 = dst->op_params[4]; const cl_uint d1 = dst->op_params[5]; - - const cl_uint cl_nb01 = nb01/ggml_type_size(src0->type); const cl_uint cl_nb02 = nb02/ggml_type_size(src0->type); const cl_uint cl_nb03 = nb03/ggml_type_size(src0->type); - const cl_uint cl_nb11 = nb11/ggml_type_size(src1->type); const cl_uint cl_nb12 = nb12/ggml_type_size(src1->type); const cl_uint cl_nb13 = nb13/ggml_type_size(src1->type); - const cl_uint cl_nb1 = nb1/ggml_type_size(dst->type); const cl_uint cl_nb2 = nb2/ggml_type_size(dst->type); const cl_uint cl_nb3 = nb3/ggml_type_size(dst->type); + const cl_uint Cout = ne03; + const cl_uint Cin = ne02; + const cl_uint N = ne13; + const cl_uint KW = ne00; + const cl_uint KH = ne01; + const cl_uint W = ne10; + const cl_uint H = ne11; + const cl_uint OW = ne0; + const cl_uint OH = ne1; + + const cl_uint s0 = dst->op_params[0]; + const cl_uint s1 = dst->op_params[1]; + const cl_uint p0 = dst->op_params[2]; + const cl_uint p1 = dst->op_params[3]; + const cl_uint d0 = dst->op_params[4]; + const cl_uint d1 = dst->op_params[5]; + + const cl_uint cl_nb00 = nb00/ggml_type_size(src0->type); + const cl_uint cl_nb01 = nb01/ggml_type_size(src0->type); + const cl_uint cl_nb02 = nb02/ggml_type_size(src0->type); + const cl_uint cl_nb03 = nb03/ggml_type_size(src0->type); + const cl_uint cl_nb10 = nb10/ggml_type_size(src1->type); + const cl_uint cl_nb11 = nb11/ggml_type_size(src1->type); + const cl_uint cl_nb12 = nb12/ggml_type_size(src1->type); + const cl_uint cl_nb13 = nb13/ggml_type_size(src1->type); + const cl_uint cl_nb1 = nb1/ggml_type_size(dst->type); + const cl_uint cl_nb2 = nb2/ggml_type_size(dst->type); + const cl_uint cl_nb3 = nb3/ggml_type_size(dst->type); const int64_t NPQ = (int64_t)N * OW * OH; @@ -17945,18 +18911,39 @@ static void ggml_cl_conv_2d(ggml_backend_t backend, const ggml_tensor * src0, co } cl_uint idx = 0; - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, idx++, shmem_size, NULL)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cout)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cin)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &N)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KW)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KH)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &W)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OW)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OH)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p1)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d1)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb01)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb02)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb03)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb11)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb12)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb13)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb2)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb3)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cout)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cin)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &N)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KW)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KH)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &W)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OW)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OH)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb00)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb01)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb02)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb03)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb10)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb11)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb12)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb13)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb2)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb3)); size_t global_work_size[] = { (size_t)NB_K * WG_K, (size_t)NB_NPQ * WG_NPQ, 1 }; size_t local_work_size[] = { (size_t)WG_K, (size_t)WG_NPQ, 1 }; @@ -18302,6 +19289,232 @@ static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_t #endif } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + + cl_context context = backend_ctx->context; + cl_kernel kernel; + + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + int M = ne01; + int N = ne1; + int K = ne00; + + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &M)); + + size_t wavesize = backend_ctx->adreno_wave_size; + size_t local_work_size[3] = { wavesize, 4, 1 }; + size_t global_work_size[3] = { (size_t)CEIL_DIV(M, 64) * 64, 4, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + const int gemm_tile_n = 64; + int N_pad = (N + gemm_tile_n - 1) & ~(gemm_tile_n - 1); + + cl_mem a_img = extra0_q4_0->q_img; + cl_mem s_img = extra0_q4_0->d_img; + GGML_ASSERT(a_img && s_img && "ILA Q4_0 weight images missing; set_tensor should have built them"); + + static const char * q4_0_bin_dp4a_env = getenv("GGML_OPENCL_Q4_0_BIN_DP4A"); + bool q4_0_bin_dp4a_on = q4_0_bin_dp4a_env + ? (atoi(q4_0_bin_dp4a_env) != 0) + : true; + // dot prod has to be available + q4_0_bin_dp4a_on = backend_ctx->has_integer_dot && q4_0_bin_dp4a_on; + + if (q4_0_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin) { + const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; + const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); + + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_mem b_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_int tb = (cl_int)((size_t)N * (K / 32)); + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_mem d_sub = nullptr; + cl_mem d_img = nullptr; + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 1, 1 }; + size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub)); + return; + } + + // Pad B through a zero-filled scratch buffer when N needs + // padding, since the GEMM kernel always reads a full N-tile. + const bool need_pad = N_pad > N; + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + if (need_pad) { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, + (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), + 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, + offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + } else { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + } + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = need_pad ? (size_t)K * N_pad : (size_t)K * N; + img_desc.buffer = need_pad ? b_padded : b_sub_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + int line_stride_matrix_A_in_bytes = M * 4; + int line_stride_matrix_S_in_bytes = M * 2; + int line_stride_matrix_B_in_bytes = K * 4; + int line_stride_matrix_C_in_bytes = M * 4; + + int c_offset_for_kernel = 0; + int b_offset_for_kernel = 0; + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &s_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &b_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_A_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_S_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_B_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_C_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); @@ -18354,6 +19567,20 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr); const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + const bool use_bin = use_q4_0_bin_kernels(backend_ctx, src0); + + if (use_bin) { + if (use_q40_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q40_MC3 is bypassed by Q4_0 binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q4_0_f32_adreno_ila(backend, src0, src1, dst); + return; + } + if (ne1 == 1 || use_q40_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; @@ -19897,6 +21124,214 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t #endif } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + + cl_context context = backend_ctx->context; + cl_kernel kernel; + + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + int M = ne01; + int N = ne1; + int K = ne00; + + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + + size_t local_work_size[3] = { 64, 8, 1 }; + size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + const int gemm_tile_n = 64; + int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + + static const char * q4_k_bin_dp4a_env = getenv("GGML_OPENCL_Q4_K_BIN_DP4A"); + bool q4_k_bin_dp4a_on = q4_k_bin_dp4a_env + ? (atoi(q4_k_bin_dp4a_env) != 0) + : true; + // dot prod has to be available + q4_k_bin_dp4a_on = backend_ctx->has_integer_dot && q4_k_bin_dp4a_on; + + if (q4_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin) { + const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; + const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); + + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_mem b_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_int tb = (cl_int)((size_t)N * (K / 32)); + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_mem d_sub = nullptr; + cl_mem d_img = nullptr; + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_int), &N)); + + size_t local_work_size[3] = { 64, 1, 1 }; + size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub)); + return; + } + + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + cl_mem b_buf = nullptr; + if (N_pad == N) { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + b_buf = b_sub_buf; + } else { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + b_buf = b_padded; + } + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N_pad; + img_desc.buffer = b_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); @@ -19949,6 +21384,20 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t // unified routes batched Q6_K lm_head to CPU). Per-layer mc3 is byte-identical. const bool use_mc3 = q4k_mc3 && (ne1 == 3) && (ne01 < 32768); + const bool use_bin = use_q4_k_bin_kernels(backend_ctx, src0); + + if (use_bin) { + if (use_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q4K_MC3 is bypassed by Q4_K binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q4_k_f32_adreno_ila(backend, src0, src1, dst); + return; + } + if (ne1 == 1 || use_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; @@ -20409,6 +21858,213 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t #endif } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q6_K_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + + cl_context context = backend_ctx->context; + cl_kernel kernel; + + cl_int err; + cl_buffer_region region; + cl_image_format img_fmt; + cl_image_desc img_desc; + + const int M = ne01; + const int N = ne1; + const int K = ne00; + + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh_img)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + + size_t local_work_size[3] = { 64, 8, 1 }; + size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } else { + const int gemm_tile_n = 64; + int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + + static const char * q6_k_bin_dp4a_env = getenv("GGML_OPENCL_Q6_K_BIN_DP4A"); + bool q6_k_bin_dp4a_on = q6_k_bin_dp4a_env + ? (atoi(q6_k_bin_dp4a_env) != 0) + : true; + // dot prod has to be available + q6_k_bin_dp4a_on = backend_ctx->has_integer_dot && q6_k_bin_dp4a_on; + + if (q6_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin) { + const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; + const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); + + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_mem b_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_int tb = (cl_int)((size_t)N * (K / 32)); + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_mem d_sub = nullptr; + cl_mem d_img = nullptr; + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 1, 1 }; + size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub)); + return; + } + + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + cl_mem b_buf = nullptr; + if (N_pad == N) { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + b_buf = b_sub_buf; + } else { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + b_buf = b_padded; + } + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N_pad; + img_desc.buffer = b_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); @@ -20457,6 +22113,20 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t // (the #1 MTP bottleneck; mc3 above can't, it reads the noshuffle layout). const bool use_q6k_tiled_mc = q6k_mc3 && (ne1 == 3) && (ne01 >= 32768) && use_q6k_tiled(backend_ctx, src0); + const bool use_bin = use_q6_k_bin_kernels(backend_ctx, src0); + + if (use_bin) { + if (use_q6k_mc3 || use_q6k_tiled_mc) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q6K_MC3 is bypassed by Q6_K binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q6_K_f32_adreno_ila(backend, src0, src1, dst); + return; + } + if (ne1 == 1 || use_q6k_mc3 || use_q6k_tiled_mc) { cl_mem ql_img = nullptr; cl_mem qh_img = nullptr; @@ -23863,10 +25533,33 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clReleaseMemObject(buf_src2)); } else { // for gemm - kernel = backend_ctx->kernel_gemm_moe_q4_0_f32_ns; - if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) { - kernel = backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin; - } + // dp4a (int8) prefill GEMM variant + static const char * q4_0_moe_dp4a_env = getenv("GGML_OPENCL_Q4_0_MOE_DP4A"); + + // It turns out that the prebuilt kernel only outperforms the dp4a variant (on X2-90) + // at very large routing counts, so we gate its use accordingly using moe_bin_min, + // which can be overridden via the GGML_OPENCL_MOE_BIN_MIN_ROUTINGS environment variable. + // The routing count is ne20 * ne21 (n_expert_used * n_tokens). + static const char * moe_bin_min_env = getenv("GGML_OPENCL_MOE_BIN_MIN_ROUTINGS"); + const int moe_bin_min = moe_bin_min_env ? atoi(moe_bin_min_env) : 4096; + + // whether bin kernels are available + const bool bin_available = backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin != nullptr; + const bool dp4a_bin_available = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin != nullptr; + + bool use_moe_dp4a = q4_0_moe_dp4a_env + ? (atoi(q4_0_moe_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E + && (dp4a_bin_available || !bin_available + || (int)(ne20 * ne21) < moe_bin_min)); + // dot prod has to be available + use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; + + const bool use_bin_kernel = bin_available && !use_moe_dp4a; + + kernel = use_bin_kernel + ? backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin + : backend_ctx->kernel_gemm_moe_q4_0_f32_ns; // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. @@ -23879,18 +25572,6 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; cl_mem buf_src2, buf_src2_emap; - // dp4a (int8) prefill GEMM variant - static const char * q4_0_moe_dp4a_env = getenv("GGML_OPENCL_Q4_0_MOE_DP4A"); - bool use_moe_dp4a = q4_0_moe_dp4a_env - ? (atoi(q4_0_moe_dp4a_env) != 0) - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); - // dot prod has to be available - use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; - // bin kernel takes precedence - if (backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin == nullptr) { - use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin == nullptr; - } - cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; @@ -23929,7 +25610,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, cl_image_desc image_desc_buf_src1; image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; - if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) { + if (use_bin_kernel) { // bin kernel uses slightly different image format image_format_buf_src1 = {CL_R, CL_FLOAT}; image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size); diff --git a/ggml/src/ggml-opencl/kernels/conv2d.cl b/ggml/src/ggml-opencl/kernels/conv2d.cl index e339c90cff59..8a04c2e597bb 100644 --- a/ggml/src/ggml-opencl/kernels/conv2d.cl +++ b/ggml/src/ggml-opencl/kernels/conv2d.cl @@ -48,8 +48,8 @@ kernel void kernel_conv_2d( uint Cout, uint Cin, uint N, uint KW, uint KH, uint W, uint H, uint OW, uint OH, uint s0, uint s1, uint p0, uint p1, uint d0, uint d1, - uint nb01, uint nb02, uint nb03, - uint nb11, uint nb12, uint nb13, + uint nb00, uint nb01, uint nb02, uint nb03, + uint nb10, uint nb11, uint nb12, uint nb13, uint nb1, uint nb2, uint nb3 ) { global T_FLOAT* knl_data = (global T_FLOAT*) ((global char*)p_knl + off_knl); @@ -95,7 +95,7 @@ kernel void kernel_conv_2d( const uint Cin_idx = crs_g / (KW*KH); const uint KH_idx = (crs_g - Cin_idx*KW*KH) / KW; const uint KW_idx = crs_g - Cin_idx*KW*KH - KH_idx*KW; - const uint knl_idx = KW_idx + KH_idx*nb01 + Cin_idx*nb02 + k_g*nb03; + const uint knl_idx = KW_idx*nb00 + KH_idx*nb01 + Cin_idx*nb02 + k_g*nb03; Ash[k_l * BS_CRS + crs_l] = knl_data[knl_idx]; } else { Ash[k_l * BS_CRS + crs_l] = (T_FLOAT)0.0f; @@ -123,7 +123,7 @@ kernel void kernel_conv_2d( const int W_idx = (int)(OW_idx * s0 + KW_idx * d0 - p0); if (H_idx >= 0 && H_idx < H && W_idx >= 0 && W_idx < W) { - const uint src_idx = W_idx + H_idx * nb11 + Cin_idx * nb12 + N_idx * nb13; + const uint src_idx = W_idx * nb10 + H_idx * nb11 + Cin_idx * nb12 + N_idx * nb13; ((T_FLOAT*)&val)[v] = src_data[src_idx]; } } diff --git a/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl b/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl index cb05637f33ac..94788e7e0f56 100644 --- a/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +++ b/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl @@ -39,8 +39,8 @@ kernel void kernel_conv_2d( uint Cout, uint Cin, uint N, uint KW, uint KH, uint W, uint H, uint OW, uint OH, uint s0, uint s1, uint p0, uint p1, uint d0, uint d1, - uint nb01, uint nb02, uint nb03, - uint nb11, uint nb12, uint nb13, + uint nb00, uint nb01, uint nb02, uint nb03, + uint nb10, uint nb11, uint nb12, uint nb13, uint nb1, uint nb2, uint nb3 ) { global half* knl_data = (global half*) ((global char*)p_knl + off_knl); @@ -86,7 +86,7 @@ kernel void kernel_conv_2d( const uint Cin_idx = crs_g / (KW*KH); const uint KH_idx = (crs_g - Cin_idx*KW*KH) / KW; const uint KW_idx = crs_g - Cin_idx*KW*KH - KH_idx*KW; - const uint knl_idx = KW_idx + KH_idx*nb01 + Cin_idx*nb02 + k_g*nb03; + const uint knl_idx = KW_idx*nb00 + KH_idx*nb01 + Cin_idx*nb02 + k_g*nb03; Ash[k_l * BS_CRS + crs_l] = knl_data[knl_idx]; } else { Ash[k_l * BS_CRS + crs_l] = (half)0.0f; @@ -114,7 +114,7 @@ kernel void kernel_conv_2d( const int W_idx = (int)(OW_idx * s0 + KW_idx * d0 - p0); if (H_idx >= 0 && H_idx < H && W_idx >= 0 && W_idx < W) { - const uint src_idx = W_idx + H_idx * nb11 + Cin_idx * nb12 + N_idx * nb13; + const uint src_idx = W_idx * nb10 + H_idx * nb11 + Cin_idx * nb12 + N_idx * nb13; ((float*)&val)[v] = src_data[src_idx]; } } diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_repack.cl b/ggml/src/ggml-opencl/kernels/flash_attn_repack.cl new file mode 100644 index 000000000000..db78d5634299 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/flash_attn_repack.cl @@ -0,0 +1,92 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +__kernel void kernel_repack_mask_for_wmm( + const global half* mask_buf, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + global half* mask_buf_padded, + const ulong mask_nb1_padded, + const ulong mask_nb2_padded, + const ulong mask_nb3_padded +) { + int col = get_global_id(0); // 0 .. n_kv + int row = get_global_id(1); // 0 .. n_q + int slice = get_global_id(2); // 0 .. (n_head * n_batch) + + int head_idx = slice % mask_ne2; + int batch_idx = slice / mask_ne2; + + ulong src_off = (ulong)batch_idx * mask_nb3 + (ulong)head_idx * mask_nb2 + (ulong)row * mask_nb1; + ulong dst_off = (ulong)batch_idx * mask_nb3_padded + (ulong)head_idx * mask_nb2_padded + (ulong)row * mask_nb1_padded; + + mask_buf_padded[dst_off / 2 + col] = mask_buf[src_off / 2 + col]; +} + +__kernel void kernel_repack_q_for_wmm( + const global float* q_buf, + const ulong q_nb1, + const ulong q_nb2, + const ulong q_nb3, + const int n_head, + __write_only image3d_t img_q_wmm +) { + int k4 = get_global_id(0); + int row = get_global_id(1); + int slice = get_global_id(2); + int batch_idx = slice / n_head; + int head_idx = slice % n_head; + + + ulong elem_off = (batch_idx * q_nb3 + head_idx * q_nb2 + row * q_nb1) / 4 + (ulong)k4 * 4; + float4 v = vload4(elem_off / 4, q_buf); + + write_imageh(img_q_wmm, (int4)(row, slice, k4, 0), convert_half4(v)); +} + +__kernel void kernel_repack_k_for_wmm( + const global half* k_buf, + const ulong k_nb1, + const ulong k_nb2, + const ulong k_nb3, + const int n_head_kv, + const int n_kv, + __write_only image3d_t img_k_wmm +) { + int kk = get_global_id(0); + int row4 = get_global_id(1); + int slice = get_global_id(2); + int batch_idx = slice / n_head_kv; + int head_kv_idx = slice % n_head_kv; + + ulong base = batch_idx * k_nb3 + head_kv_idx * k_nb2; + int row0 = row4 * 4; + half4 v; + v.x = (row0 + 0 < n_kv) ? k_buf[(base + (ulong)(row0 + 0) * k_nb1) / 2 + kk] : (half)0; + v.y = (row0 + 1 < n_kv) ? k_buf[(base + (ulong)(row0 + 1) * k_nb1) / 2 + kk] : (half)0; + v.z = (row0 + 2 < n_kv) ? k_buf[(base + (ulong)(row0 + 2) * k_nb1) / 2 + kk] : (half)0; + v.w = (row0 + 3 < n_kv) ? k_buf[(base + (ulong)(row0 + 3) * k_nb1) / 2 + kk] : (half)0; + + write_imageh(img_k_wmm, (int4)(kk, row4, slice, 0), v); +} + +__kernel void kernel_repack_v_for_wmm( + const global half* v_buf, + const ulong v_nb1, + const ulong v_nb2, + const ulong v_nb3, + const int n_head_kv, + __write_only image3d_t img_v_wmm +) { + int hdim4 = get_global_id(0); // now fastest — walks contiguous memory + int row = get_global_id(1); + int slice = get_global_id(2); + int batch_idx = slice / n_head_kv; + int head_kv_idx = slice % n_head_kv; + + ulong row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + (ulong)row * v_nb1; + half4 v = vload4((row_off / 2 + (ulong)hdim4 * 4) / 4, v_buf); + + write_imageh(img_v_wmm, (int4)(row, hdim4, slice, 0), v); +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl new file mode 100644 index 000000000000..565285f4b293 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl @@ -0,0 +1,137 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK4_0 32 +#define N_SIMDGROUP 4 + +#define dequantizeBlockAccum_ila_1row_hi(total_sum, bits4, scale, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sum += ((bits4.s0 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sum += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sum += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sum += (((bits4.s0 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sum += ((bits4.s1 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sum += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sum += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sum += (((bits4.s1 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sum += ((bits4.s2 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sum += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sum += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sum += (((bits4.s2 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sum += ((bits4.s3 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sum += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sum += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sum += (((bits4.s3 & 0xF000) >> 12) - 8) * scale * shared_y; + +#define dequantizeBlockAccum_ila_1row_lo(total_sum, bits4, scale, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sum += ((bits4.s4 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sum += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sum += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sum += (((bits4.s4 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sum += ((bits4.s5 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sum += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sum += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sum += (((bits4.s5 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sum += ((bits4.s6 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sum += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sum += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sum += (((bits4.s6 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sum += ((bits4.s7 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sum += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sum += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sum += (((bits4.s7 & 0xF000) >> 12) - 8) * scale * shared_y; + + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q4_0_f32_32b_trans( + __read_only image1d_buffer_t src0_q, + global half * src0_d, + __read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + __private uint4 regA; + __private half regS; + __private float8 regB; + __private float totalSum = 0.0f; + + for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) { + regS = src0_d[k * M + gid]; + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + regA.s0 = read_imageui(src0_q, ((k * 4 + 0) * M + gid)).x; + regA.s1 = read_imageui(src0_q, ((k * 4 + 1) * M + gid)).x; + regA.s2 = read_imageui(src0_q, ((k * 4 + 2) * M + gid)).x; + regA.s3 = read_imageui(src0_q, ((k * 4 + 3) * M + gid)).x; + + dequantizeBlockAccum_ila_1row_hi(totalSum, as_ushort8(regA), regS, regB); + dequantizeBlockAccum_ila_1row_lo(totalSum, as_ushort8(regA), regS, regB); + } + + __local float reduceLM[SIMDGROUP_WIDTH * 3]; + if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = totalSum; + if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = totalSum; + if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = totalSum; + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + if (groupId == 0) { + dst = (global float*)((global char*)dst + offsetd); + if (gid < M) { + dst[gid] = totalSum; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl new file mode 100644 index 000000000000..2dbd943fda29 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_K 256 +#define K_SCALE_SIZE 12 +#define N_SIMDGROUP 8 +#define SIMDGROUP_WIDTH 64 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uint stride, + uchar * d, + uchar * m +) { + if (j < 4) { + *d = q[j*stride] & 63; + *m = q[(j+4)*stride] & 63; + } else { + *d = (q[(j+4)*stride] & 0x0F) | ((q[(j-4)*stride] & 0xC0) >> 2); + *m = ((q[(j+4)*stride] >> 4) & 0x0F) | ((q[j*stride] & 0xC0) >> 2); + } +} + +static inline float8 q4_k_to_fp32_packed8(ushort2 q4x8, float scale, float minv) { + float8 fp32x8; + fp32x8.s0 = (q4x8.s0 & 0x000F) * scale - minv; + fp32x8.s1 = ((q4x8.s0 & 0x00F0) >> 4) * scale - minv; + fp32x8.s2 = ((q4x8.s0 & 0x0F00) >> 8) * scale - minv; + fp32x8.s3 = ((q4x8.s0 & 0xF000) >> 12) * scale - minv; + fp32x8.s4 = (q4x8.s1 & 0x000F) * scale - minv; + fp32x8.s5 = ((q4x8.s1 & 0x00F0) >> 4) * scale - minv; + fp32x8.s6 = ((q4x8.s1 & 0x0F00) >> 8) * scale - minv; + fp32x8.s7 = ((q4x8.s1 & 0xF000) >> 12) * scale - minv; + return fp32x8; +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void gemv_noshuffle_q4_k_f32_32b_trans( + read_only image1d_buffer_t src0_q, + __global half * src0_d, + __global half * src0_dm, + __global uchar * src0_s, + __read_only image1d_buffer_t src1, + __global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + uint i01 = get_global_id(0); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + int num_subblocks = ne00 / 32; + + __private float sum = 0.0f; + + // Loop over sub-blocks of 32 elements, N_SIMDGROUP sub-blocks per iter + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; + uint j = ib % 8; + + // Load d and dmin for this super-block + half d_val = src0_d[sb * ne01 + i01]; + half dm_val = src0_dm[sb * ne01 + i01]; + + // Load sub-block scale and min. s is transposed [nb][12][M]; stride ne01 per code. + global const uchar * sc = src0_s + sb * K_SCALE_SIZE * ne01 + i01; + uchar sv, mn; + get_scale_min_k4(j, sc, ne01, &sv, &mn); + + float scale = (float)d_val * (float)sv; + float minv = (float)dm_val * (float)mn; + + // Load 4 uints of quants (32 nibbles = 32 elements), column-major stride ne01 + uint q_base = ib * ne01 * 4 + i01; + + uint4 regQ; + regQ.s0 = read_imageui(src0_q, q_base).x; + regQ.s1 = read_imageui(src0_q, q_base + ne01).x; + regQ.s2 = read_imageui(src0_q, q_base + ne01 * 2).x; + regQ.s3 = read_imageui(src0_q, q_base + ne01 * 3).x; + + // Load activations: 32 floats = 8 float4s + uint y_offset = ib * 8; + + float4 y_local = (slid < 8) ? read_imagef(src1, (y_offset + slid)) : (float4)0.0f; + float4 y0 = sub_group_broadcast(y_local, 0); + float4 y1 = sub_group_broadcast(y_local, 1); + float4 y2 = sub_group_broadcast(y_local, 2); + float4 y3 = sub_group_broadcast(y_local, 3); + float4 y4 = sub_group_broadcast(y_local, 4); + float4 y5 = sub_group_broadcast(y_local, 5); + float4 y6 = sub_group_broadcast(y_local, 6); + float4 y7 = sub_group_broadcast(y_local, 7); + + float8 fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s0), scale, minv); + float4 acc = y0 * fp32x8.lo; + acc += y1 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s1), scale, minv); + acc += y2 * fp32x8.lo; + acc += y3 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s2), scale, minv); + acc += y4 * fp32x8.lo; + acc += y5 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s3), scale, minv); + acc += y6 * fp32x8.lo; + acc += y7 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory over N_SIMDGROUP subgroups + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid > 0) { + reduceLM[SIMDGROUP_WIDTH * (sgid - 1) + slid] = sum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) { + for (uint i = 0; i < N_SIMDGROUP - 1; ++i) { + sum += reduceLM[SIMDGROUP_WIDTH * i + slid]; + } + } + + // 1 output per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_32b_trans.cl new file mode 100644 index 000000000000..2e1e2d76dc8d --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_32b_trans.cl @@ -0,0 +1,128 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_K 256 +#define N_SIMDGROUP 8 +#define SIMDGROUP_WIDTH 64 + +static inline float8 q6_k_to_fp32_packed8(ushort2 ql8, ushort qh8, float d_scale) { + float8 fp32x8; + fp32x8.s0 = ((float)(( ql8.s0 & 0x000F) | ((uint)((qh8 ) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s1 = ((float)((( ql8.s0 >> 4) & 0x000F) | ((uint)((qh8 >> 2) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s2 = ((float)((( ql8.s0 >> 8) & 0x000F) | ((uint)((qh8 >> 4) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s3 = ((float)((( ql8.s0 >> 12)& 0x000F) | ((uint)((qh8 >> 6) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s4 = ((float)(( ql8.s1 & 0x000F) | ((uint)((qh8 >> 8) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s5 = ((float)((( ql8.s1 >> 4) & 0x000F) | ((uint)((qh8 >>10) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s6 = ((float)((( ql8.s1 >> 8) & 0x000F) | ((uint)((qh8 >>12) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s7 = ((float)((( ql8.s1 >> 12)& 0x000F) | ((uint)((qh8 >>14) & 0x3) << 4)) - 32.f) * d_scale; + return fp32x8; +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_noshuffle_q6_k_f32_32b_trans( + __read_only image1d_buffer_t src0_ql, + __read_only image1d_buffer_t src0_qh, + __global char * src0_s, + __global half * src0_d, + __read_only image1d_buffer_t src1, + __global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + uint i01 = get_global_id(0); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + int num_superblocks = ne00 / QK_K; + int num_subblocks = ne00 / 32; // 2 sub-blocks of 16 processed per iter below + int scales_per_row = num_superblocks * 16; + + __private float sum = 0.0f; + + // Loop over 32-element groups (2 sub-blocks of 16 each), N_SIMDGROUP groups per iter. + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; // super-block index + uint j = ib % 8; // 32-element group within super-block (0..7) + + // Load d for this super-block. + half d_val = src0_d[sb * ne01 + i01]; + + // Load 2 sub-block scales (int8), one per 16 elements. + global const char * sc = src0_s + i01 * scales_per_row + sb * 16; + float scale0 = (float)d_val * (float)sc[j * 2]; + float scale1 = (float)d_val * (float)sc[j * 2 + 1]; + + // Load 4 uints of ql (32 elements, 4-bit each = 128 bits), column-major stride ne01. + uint ql_base = (ib * 4) * ne01 + i01; + uint4 regQL; + regQL.s0 = read_imageui(src0_ql, ql_base).x; + regQL.s1 = read_imageui(src0_ql, ql_base + ne01).x; + regQL.s2 = read_imageui(src0_ql, ql_base + ne01 * 2).x; + regQL.s3 = read_imageui(src0_ql, ql_base + ne01 * 3).x; + + // Load 2 uints of qh (32 elements, 2-bit each = 64 bits), column-major stride ne01. + uint qh_base = (ib * 2) * ne01 + i01; + uint2 regQH; + regQH.s0 = read_imageui(src0_qh, qh_base).x; + regQH.s1 = read_imageui(src0_qh, qh_base + ne01).x; + + // Load activations: 32 floats = 8 float4s. + uint y_offset = ib * 8; + + float4 y_local = (slid < 8) ? read_imagef(src1, (y_offset + slid)) : (float4)0.0f; + float4 y0 = sub_group_broadcast(y_local, 0); + float4 y1 = sub_group_broadcast(y_local, 1); + float4 y2 = sub_group_broadcast(y_local, 2); + float4 y3 = sub_group_broadcast(y_local, 3); + float4 y4v = sub_group_broadcast(y_local, 4); + float4 y5 = sub_group_broadcast(y_local, 5); + float4 y6 = sub_group_broadcast(y_local, 6); + float4 y7 = sub_group_broadcast(y_local, 7); + + // Dequantize elements 0..7 (scale0). + float8 fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s0), (ushort)(regQH.s0 & 0xFFFF), scale0); + + float4 acc = y0 * fp32x8.lo; + acc += y1 * fp32x8.hi; + + // Dequantize elements 8..15 (scale0). + fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s1), (ushort)(regQH.s0 >> 16), scale0); + + acc += y2 * fp32x8.lo; + acc += y3 * fp32x8.hi; + + // Dequantize elements 16..23 (scale1). + fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s2), (ushort)(regQH.s1 & 0xFFFF), scale1); + + acc += y4v * fp32x8.lo; + acc += y5 * fp32x8.hi; + + // Dequantize elements 24..31 (scale1). + fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s3), (ushort)(regQH.s1 >> 16), scale1); + + acc += y6 * fp32x8.lo; + acc += y7 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory, assumes #subgroups=4 + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid > 0) { + reduceLM[SIMDGROUP_WIDTH * (sgid - 1) + slid] = sum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) { + for (uint i = 0; i < N_SIMDGROUP - 1; ++i) { + sum += reduceLM[SIMDGROUP_WIDTH * i + slid]; + } + } + + // 1 output per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl b/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl index e6295c81648e..2f5c110bf7e7 100644 --- a/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +++ b/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl @@ -20,11 +20,13 @@ kernel void kernel_moe_reorder_b( uint router_idx = router[post_router_idx]; - float4 out = (float4)(0); - if (router_idx != 0xFFFFFFFF) { - ushort activation_idx = router_idx / map_ratio; - out = src[activation_idx * K / 4 + k_4]; + // Padded slots need not be written at all. The MoE GEMMs accumulate per output + // column and scatter only the real columns, so whatever sits in a padded slot + // never reaches dst + if (router_idx == 0xFFFFFFFF) { + return; } - dst[post_router_idx * K / 4 + k_4] = out; + ushort activation_idx = router_idx / map_ratio; + dst[post_router_idx * K / 4 + k_4] = src[activation_idx * K / 4 + k_4]; } diff --git a/ggml/src/ggml-opencl/kernels/ssm_scan.cl b/ggml/src/ggml-opencl/kernels/ssm_scan.cl index 37698d123f41..1889b74cdb92 100644 --- a/ggml/src/ggml-opencl/kernels/ssm_scan.cl +++ b/ggml/src/ggml-opencl/kernels/ssm_scan.cl @@ -214,3 +214,133 @@ kernel void kernel_ssm_scan_f32_mamba2_d256( s_warp[tid + 128] = state2; s_warp[tid + 192] = state3; } + +kernel void kernel_ssm_scan_f32( + global const char * s_buf, + ulong s_off, + global const char * x_buf, + ulong x_off, + global const char * dt_buf, + ulong dt_off, + global const char * A_buf, + ulong A_off, + global const char * B_buf, + ulong B_off, + global const char * C_buf, + ulong C_off, + global const char * ids_buf, + ulong ids_off, + global char * dst_buf, + ulong dst_off, + ulong s_nb2, + ulong s_nb3, + ulong x_nb2, + ulong x_nb3, + ulong dt_nb1, + ulong dt_nb2, + ulong A_nb1, + ulong B_nb2, + ulong B_nb3, + ulong C_nb2, + ulong C_nb3, + ulong state_off, + int head_dim, + int n_head, + int n_group, + int n_tokens, + ulong s_nb1, + ulong x_nb1, + ulong B_nb1, + ulong C_nb1, + uint A_ne0, + uint d_state, + uint n_seqs, + uint K, + local float * reduce +) { + global const char * s_data = s_buf + s_off; + global const char * x_data = x_buf + x_off; + global const char * dt_data = dt_buf + dt_off; + global const char * A_data = A_buf + A_off; + global const char * B_data = B_buf + B_off; + global const char * C_data = C_buf + C_off; + global const int * ids_data = (global const int *) (ids_buf + ids_off); + global float * dst = (global float *) (dst_buf + dst_off); + const uint y_elems = state_off / sizeof(float); + + const uint tid = get_local_id(0); + const uint inner_idx = get_group_id(0); + const uint seq_idx = get_group_id(1); + const uint head_idx = inner_idx / head_dim; + const uint dim_idx = inner_idx - head_idx * head_dim; + const uint group_idx = head_idx / (n_head / n_group); + const uint state_slot = (uint) ids_data[seq_idx]; + + const ulong s_idx = (ulong) state_slot * s_nb3 + + (ulong) head_idx * s_nb2 + + (ulong) dim_idx * s_nb1 + + (ulong) tid * sizeof(float); + float state = *((global const float *) (s_data + s_idx)); + + const ulong A_idx = (ulong) head_idx * A_nb1 + + (ulong) (tid % A_ne0) * sizeof(float); + const float A_value = *((global const float *) (A_data + A_idx)); + + for (int token_idx = 0; token_idx < n_tokens; ++token_idx) { + const ulong x_idx = (ulong) head_idx * x_nb1 + + (ulong) token_idx * x_nb2 + + (ulong) seq_idx * x_nb3 + + (ulong) dim_idx * sizeof(float); + const ulong dt_idx = (ulong) token_idx * dt_nb1 + + (ulong) seq_idx * dt_nb2 + + (ulong) head_idx * sizeof(float); + const ulong B_idx = (ulong) group_idx * B_nb1 + + (ulong) token_idx * B_nb2 + + (ulong) seq_idx * B_nb3 + + (ulong) tid * sizeof(float); + const ulong C_idx = (ulong) group_idx * C_nb1 + + (ulong) token_idx * C_nb2 + + (ulong) seq_idx * C_nb3 + + (ulong) tid * sizeof(float); + + const float x_value = *((global const float *) (x_data + x_idx)); + const float dt_value = *((global const float *) (dt_data + dt_idx)); + const float B_value = *((global const float *) (B_data + B_idx)); + const float C_value = *((global const float *) (C_data + C_idx)); + const float dt_soft_plus = dt_value > 20.0f ? dt_value : log(1.0f + exp(dt_value)); + const float dA = exp(dt_soft_plus * A_value); + const float x_dt = x_value * dt_soft_plus; + + state = mad(state, dA, B_value * x_dt); + reduce[tid] = state * C_value; + barrier(CLK_LOCAL_MEM_FENCE); + + for (uint stride = d_state / 2; stride > 0; stride >>= 1) { + if (tid < stride) { + reduce[tid] += reduce[tid + stride]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (tid == 0) { + const uint y_idx = dim_idx + head_idx * head_dim + + token_idx * n_head * head_dim + + seq_idx * n_tokens * n_head * head_dim; + dst[y_idx] = reduce[0]; + } + + const uint snapshot_slot = n_tokens - 1 - token_idx; + if (snapshot_slot > 0 && snapshot_slot < K) { + const uint snapshot_idx = y_elems + tid + dim_idx * d_state + + head_idx * d_state * head_dim + + (snapshot_slot * n_seqs + seq_idx) * d_state * head_dim * n_head; + dst[snapshot_idx] = state; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const uint state_idx = y_elems + tid + dim_idx * d_state + + head_idx * d_state * head_dim + + seq_idx * d_state * head_dim * n_head; + dst[state_idx] = state; +} diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 006e005cb7aa..cd06b22e8a78 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -117,16 +117,7 @@ bool is_same_shape(const ggml_tensor * a, const ggml_tensor * b) { bool is_conv_states_all_tensor(const ggml_tensor * tensor) { return tensor != nullptr && strncmp(tensor->name, "conv_states_all", strlen("conv_states_all")) == 0; } - -// CPY writing the tail of conv_input (the concat of the previous conv state and the new tokens) -// back into a slot block of the recurrent state cache. Detected structurally because the rollback -// variant (cparams.n_rs_seq > 0) emits one such CPY per snapshot slot without naming them. -bool is_conv_state_writeback(const ggml_tensor * node) { - return node->op == GGML_OP_CPY && node->view_src != nullptr && GgmlOvDecoder::is_kvcache(node->view_src, nullptr) && - node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr && - node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && - node->src[1]->view_src == node->view_src; -} +} // namespace // MoE expert aggregation (build_moe_ffn in llama-graph.cpp): each expert plane is // `ggml_view_2d(experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1])` and the planes @@ -174,20 +165,31 @@ bool is_moe_expert_sum_add(const ggml_tensor * node) { return base != nullptr && base->ne[1] > 1 && plane_indices.size() == static_cast<size_t>(base->ne[1]); } -} // namespace -static std::string get_tensor_ov_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) { +std::string GgmlOvDecoder::get_tensor_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) { if (tensor == nullptr) { return ""; } - const size_t hash_pos = ggml_hash_find(&cgraph->visited_hash_set, tensor); - if (((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || GgmlOvDecoder::is_kvcache(tensor, nullptr)) && - hash_pos != GGML_HASHSET_FULL && ggml_bitset_get(cgraph->visited_hash_set.used, hash_pos)) { - return std::string(tensor->name) + "#" + std::to_string(hash_pos); + if ((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || is_kvcache(tensor, nullptr)) { + // Hash-table slots depend on tensor addresses and differ between contexts. + // Graph ordinals disambiguate duplicate names while keeping compiled-model + // ports identical for equivalent graphs in different contexts. + const auto * node = std::find(cgraph->nodes, cgraph->nodes + cgraph->n_nodes, tensor); + if (node != cgraph->nodes + cgraph->n_nodes) { + return std::string(tensor->name) + "#n" + std::to_string(node - cgraph->nodes); + } + const auto * leaf = std::find(cgraph->leafs, cgraph->leafs + cgraph->n_leafs, tensor); + if (leaf != cgraph->leafs + cgraph->n_leafs) { + return std::string(tensor->name) + "#l" + std::to_string(leaf - cgraph->leafs); + } } return tensor->name; } +static std::string get_tensor_ov_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) { + return GgmlOvDecoder::get_tensor_name(cgraph, tensor); +} + static std::string get_tensor_graph_input_ov_name(const GgmlOvDecoder * decoder, const ggml_cgraph * cgraph, const ggml_tensor * tensor, @@ -198,8 +200,20 @@ static std::string get_tensor_graph_input_ov_name(const GgmlOvDecoder * decoder, if (GgmlOvDecoder::is_inp_emb(tensor, op)) { return "embd"; } - if (decoder->is_stateful() && GgmlOvDecoder::is_inp_mask(tensor, op)) { - return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa"; + if (GgmlOvDecoder::is_inp_mask(tensor, op)) { + // Give the two attention masks distinct OV parameter names. build_attn_inp_kq_mask() + // names the full-attention mask and the sliding-window mask identically, so keying a + // parameter off the name alone makes the second mask overwrite the first and both + // attention types read one parameter. Tell them apart by tensor identity, using the + // SWA classification computed in compute_llm_params(). An empty swa_layers set means + // there is only one mask in play and the plain name is correct. + const bool is_swa = decoder->is_swa_mask(tensor); + if (decoder->is_stateful()) { + return is_swa ? "self_kq_mask_swa" : "self_kq_mask"; + } + if (is_swa) { + return get_tensor_ov_name(cgraph, tensor) + "_swa"; + } } return get_tensor_ov_name(cgraph, tensor); } @@ -231,7 +245,7 @@ void GgmlOvDecoder::set_input_output() { if (src->op == GGML_OP_VIEW) { // Traverse upward through nested VIEW operations std::remove_reference_t<decltype(current_node_info.node_inputs_views[src_name])> view_chain; - auto current = src; + auto * current = src; while (current != nullptr) { auto current_name = get_tensor_ov_name(m_cgraph, current); @@ -318,9 +332,7 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { break; } case GGML_OP_MUL_MAT: { - if (node->src[0]->op == GGML_OP_VIEW && node->src[1]->op == GGML_OP_VIEW) { - op_case = 3; - } else if (node->src[1]->op == GGML_OP_SOFT_MAX) { + if (node->src[1]->op == GGML_OP_SOFT_MAX) { // In the case of `-fa off`, softmax is used, v_trans=true, the dynamic dim is ne[0] for cache_v op_case = 2; } @@ -441,7 +453,7 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { if (node->src[0]->op == GGML_OP_VIEW) { if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) { op_case = 1; - } else if (is_conv_state_writeback(node)) { + } else if (GgmlOvDecoder::is_conv_state_writeback(node)) { op_case = 2; break; } else if (is_conv_states_all_tensor(node->view_src) && node->src[1] != nullptr && @@ -532,6 +544,40 @@ std::optional<int> extract_layer_from_name(const std::string & name) { return layer; } +// Recover the sliding window width from ggml's own SWA mask. llama.cpp never passes n_swa to a +// backend, but fill_mask() writes it into the mask: a query row keeps exactly the cells inside +// its window, so the widest row counts min(pos + 1, n_swa) unmasked cells. Counting rather than +// looking for a contiguous band is what makes this work on the KV-cache mask, where columns are +// physical cache cells in arbitrary order, not positions. +// Assumes LLAMA_SWA_TYPE_STANDARD, the only type the caller reconstructs. +static int get_swa_window_from_mask(const ggml_tensor * mask) { + if (mask->data == nullptr || !ggml_backend_buffer_is_host(mask->buffer)) { + return -1; + } + if (mask->type != GGML_TYPE_F16 && mask->type != GGML_TYPE_F32) { + return -1; + } + + const int64_t n_kv = mask->ne[0]; + const int64_t n_tokens = mask->ne[1]; + int64_t window = 0; + + for (int64_t r = 0; r < n_tokens; r++) { + int64_t kept = 0; + for (int64_t c = 0; c < n_kv; c++) { + const size_t i = (size_t) r * n_kv + c; + const float v = mask->type == GGML_TYPE_F16 ? ggml_fp16_to_fp32(((const ggml_fp16_t *) mask->data)[i]) : + ((const float *) mask->data)[i]; + if (v > -INFINITY) { + kept++; + } + } + window = std::max(window, kept); + } + + return window > 0 ? (int) window : -1; +} + std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgraph * cgraph, bool is_static) { ModelParams model_params; ComputeParams compute_params; @@ -566,9 +612,8 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr if (node->src[1]->view_src != nullptr) { if (node->src[3] != nullptr) { return 4; // decoder self-attention - } else { - return 5; // cross-attention or encoder self-attention - }; + } + return 5; // cross-attention or encoder self-attention } break; default: @@ -597,10 +642,100 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr return -1; }; + // Resolve the attention mask an attention node consumes, mirroring the src layout that + // get_attention_pattern_case() classifies. Used by the SWA pre-pass below. + auto get_attention_op_mask = [&get_attention_pattern_case](const ggml_tensor * node) -> const ggml_tensor * { + switch (get_attention_pattern_case(node)) { + case 0: + case 1: + return node->src[3]; + case 2: + case 3: + return node->src[1]; + default: + return nullptr; + } + }; + + // Pre-pass: classify sliding-window vs full-attention layers. + // + // An interleaved-SWA model keeps two KV caches and two attention masks, and hands each layer + // whichever pair matches its attention type. The mask tensor does not say which is which: both + // are named "attn_inp_kq_mask" by build_attn_inp_kq_mask(), and both carry the same n_kv because + // llama_kv_cache::get_n_kv() pads occupancy up to a common multiple. + // + // The KV cache does say. Each cache allocates cache_k_l<N> once at load time with its own cell + // count: the windowed cache is sized from the window + // (PAD(min(size_base, n_swa*(unified ? n_seq_max : 1) + n_ubatch), 256), see + // llama_kv_cache_iswa), the full-attention one spans the whole context. Read the LEAF buffer + // behind the VIEW rather than the VIEW itself: the leaf extent is a constant per layer, known + // from the first graph onwards, while the view grows with context depth and would invert the + // comparison at shallow depth. + // + // Layers whose leaf is smaller than the largest leaf are the windowed ones. When every layer + // reports the same extent there is no distinction to draw -- either the model has no windowed + // layers, or the window is at least as large as the context so the two caches coincide, in + // which case a windowed layer and a full-attention one compute the same thing. + // + // Getting this wrong is silent and severe: with the windowed layers classified as + // full-attention, permute's KV slicing uses attention_size instead of attention_size_swa. The + // two agree while the context is shorter than the window, then diverge, and the mask add fails + // shape inference ("Failed to broadcast-merge input shapes") partway into a long prompt. + { + std::map<int, int64_t> layer_extent; // layer -> leaf cache_k cell count + std::map<int, const ggml_tensor *> layer_mask; // layer -> mask it consumes + int64_t max_extent = 0; + + for (int i = 0; i < cgraph->n_nodes; i++) { + const ggml_tensor * mask = get_attention_op_mask(cgraph->nodes[i]); + if (mask == nullptr) { + continue; + } + const ggml_tensor * cache_k_permute = nullptr; + switch (get_attention_pattern_case(cgraph->nodes[i])) { + case 0: cache_k_permute = cgraph->nodes[i]->src[1]; break; + case 1: cache_k_permute = cgraph->nodes[i]->src[1]->src[0]; break; + case 2: cache_k_permute = cgraph->nodes[i]->src[0]->src[0]; break; + default: cache_k_permute = cgraph->nodes[i]->src[0]->src[0]->src[0]; break; + } + const ggml_tensor * cache_k_view = cache_k_permute->src[0]; + if (cache_k_view->op != GGML_OP_VIEW) { + continue; + } + const ggml_tensor * leaf = cache_k_view->src[0]; + auto layer = extract_layer_from_name(leaf->name); + if (!layer.has_value()) { + continue; + } + layer_extent[layer.value()] = leaf->ne[1]; + layer_mask[layer.value()] = mask; + max_extent = std::max(max_extent, leaf->ne[1]); + } + + for (const auto & [layer, extent] : layer_extent) { + if (extent < max_extent) { + model_params.swa_layers.push_back(layer); + if (model_params.swa_mask == nullptr) { + model_params.swa_mask = layer_mask[layer]; + } + } + } + std::sort(model_params.swa_layers.begin(), model_params.swa_layers.end()); + + if (ggml_openvino_getenv_int("GGML_OPENVINO_LOG_SWA_LAYERS")) { + std::string per_layer; + for (const auto & [layer, extent] : layer_extent) { + per_layer += " " + std::to_string(layer) + ":" + std::to_string(extent) + + (extent < max_extent ? "(swa)" : ""); + } + GGML_LOG_WARN("ov-swa: attn_layers=%zu max_extent=%ld swa_layers=%zu |%s\n", layer_extent.size(), + (long) max_extent, model_params.swa_layers.size(), per_layer.c_str()); + } + } + bool rope_seen = false; for (int i = 0; i < cgraph->n_nodes; i++) { - auto * node = cgraph->nodes[i]; - std::string name = std::string(node->name); + ggml_tensor * node = cgraph->nodes[i]; const int attention_pattern_case = get_attention_pattern_case(node); if (attention_pattern_case != -1) { ggml_tensor * cache_k_permute = nullptr; @@ -654,11 +789,14 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr ggml_tensor * cache_k = cache_k_view->src[0]; int layer = extract_layer_from_name(cache_k->name).value(); - std::string mask_name(mask->name); + // Classified by the pre-pass above, which groups layers by mask tensor identity. The + // mask NAME cannot be used: build_attn_inp_kq_mask() gives both masks the same name. + const bool layer_is_swa = std::find(model_params.swa_layers.begin(), model_params.swa_layers.end(), + layer) != model_params.swa_layers.end(); model_params.kv_buffer_ctx_id = ggml_backend_openvino_buffer_get_ctx_id(cache_k->buffer); - if (mask_name.find("swa") != std::string::npos) { - model_params.swa_layers.push_back(layer); + model_params.n_heads_kv_per_layer[layer] = cache_k_permute->ne[2]; + if (layer_is_swa) { model_params.ctx_per_seq_swa = cache_k->ne[1]; } else { model_params.ctx_per_seq = cache_k->ne[1]; @@ -671,8 +809,9 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr memcpy(&offset, cache_k_view->op_params, sizeof(size_t)); compute_params.seq_active_start = offset / seq_size; - if (mask_name.find("swa") != std::string::npos) { + if (layer_is_swa) { compute_params.attention_size_swa = mask->ne[0]; + compute_params.swa_window = get_swa_window_from_mask(mask); } else { compute_params.attention_size = mask->ne[0]; } @@ -708,11 +847,11 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr // mixed SWA/non-SWA layers with different n_dims or freq_base), we cannot // share a single precomputed rope_sin/rope_cos. Track divergence so the // translator falls back to per-op make_sin_cos in that case. - static_assert(sizeof(model_params.rope_params) == sizeof(int32_t) * 15, "rope_params size"); + static_assert(sizeof(model_params.rope_params) == sizeof(int32_t) * 16, "rope_params size"); if (!rope_seen) { - memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 15); + memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 16); rope_seen = true; - } else if (memcmp(model_params.rope_params, node->op_params, sizeof(int32_t) * 15) != 0) { + } else if (memcmp(model_params.rope_params, node->op_params, sizeof(int32_t) * 16) != 0) { model_params.mixed_rope_params = true; } } @@ -752,8 +891,41 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr } } } + if (model_params.n_heads_kv == -1) { + for (int i = 0; i < cgraph->n_nodes; i++) { + const auto * node = cgraph->nodes[i]; + const ggml_tensor * mask = nullptr; + if (node->op == GGML_OP_SOFT_MAX) { + mask = node->src[1]; + } else if (node->op == GGML_OP_FLASH_ATTN_EXT) { + mask = node->src[3]; + } else { + continue; + } + if (mask == nullptr || mask->op != GGML_OP_NONE || !(mask->flags & GGML_TENSOR_FLAG_INPUT) || + node->src[0] == nullptr) { + continue; + } + model_params.is_cacheless_attn = true; + model_params.n_seq = 1; + model_params.ctx_per_seq = mask->ne[0]; + compute_params.input_len = node->src[0]->ne[1]; + compute_params.token_len_per_seq = compute_params.input_len; + break; + } + } + auto * output_tensor = cgraph->nodes[cgraph->n_nodes - 1]; compute_params.output_len = output_tensor->ne[1]; + if (model_params.is_cacheless_attn) { + for (int i = 0; i < cgraph->n_nodes; i++) { + const auto * node = cgraph->nodes[i]; + if (node->op == GGML_OP_GET_ROWS && is_output_idx(node->src[1], node)) { + compute_params.output_len = node->src[1]->ne[0]; + break; + } + } + } // for NPU, output_len is always 1 except for llama-perplexity if (is_static && compute_params.output_len == 0) { compute_params.output_len = 1; @@ -774,7 +946,6 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, if (m_naive) { return input != nullptr ? ov::PartialShape{get_shape(input)} : ov::PartialShape{get_shape(op)}; } - auto name = std::string(input->name); ov::PartialShape input_shape; if (is_inp_tok(input, op) || is_inp_pos(input, op)) { @@ -790,6 +961,10 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, // output index input_shape = ov::PartialShape{1, 1, 1, m_is_static ? m_compute_params.output_len : -1}; + } else if (is_inp_mean(input, op)) { + input_shape = m_is_static ? ov::PartialShape{1, 1, input->ne[1], m_prefill_chunk_size} : + ov::PartialShape{1, 1, -1, -1}; + } else if (is_inp_mask(input, op)) { // mask if (m_is_static) { @@ -814,11 +989,19 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, if (is_stateful() && !is_flat_kv) { // Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size] // to stateful layout [1, seq, n_heads_kv, head_size]. + // NOTE: Gemma4 uses per-layer-type KV shapes, so no single scalar describes every + // layer. E2B varies only the head size (sliding 256, full 512); 12B also varies the + // head COUNT (sliding 8 x 256, full 1 x 512). Take the head count for this tensor's + // own layer type and derive the head size from its own combined dim, so both layer + // types get the correct split. Using the model-level count split 12B's sliding + // states as 1 x 2048 and decoded garbage. assert(input_shape.size() == 4 && input_shape[0] == 1 && input_shape[1] == 1 && - input_shape[2].is_dynamic() && - input_shape[3] == (m_model_params.n_heads_kv * m_model_params.head_size)); - input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv, - m_model_params.head_size}; + input_shape[2].is_dynamic() && input_shape[3].is_static()); + const int n_heads_kv = get_n_heads_kv_for_tensor(input); + assert(n_heads_kv > 0 && input_shape[3].get_length() % n_heads_kv == 0); + const int64_t combined_dim = input_shape[3].get_length(); // n_heads_kv * head_size + const int64_t head_size = combined_dim / n_heads_kv; + input_shape = {input_shape[0], ov::Dimension::dynamic(), n_heads_kv, head_size}; } } else if (is_kv_idx(input, op)) { @@ -840,8 +1023,14 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, if (op->op == GGML_OP_SOFT_MAX && op->src[1] != nullptr && op->src[1]->op == GGML_OP_NONE && op->src[1]->flags & GGML_TENSOR_FLAG_INPUT && op->src[1] == input) { // for softmax input mask, the shape is [1, 1, seq_active, seq_active], where seq_active is determined by the input active sequence length instead of the kv cache sequence length - input_shape[2] = -1; - input_shape[3] = -1; + if (m_is_static) { + const int64_t seq_active = m_is_prefill ? m_prefill_chunk_size : 1; + input_shape[2] = seq_active; + input_shape[3] = seq_active; + } else { + input_shape[2] = -1; + input_shape[3] = -1; + } } return input_shape; } @@ -894,6 +1083,10 @@ void GgmlOvDecoder::add_extra_inputs() { if (m_compute_params.attention_size_swa != -1) { create_1d_input("attention_size_swa", m_compute_params.attention_size_swa); } + // only the stateful SWA mask consumes this + if (is_stateful() && m_compute_params.swa_window != -1) { + create_1d_input("swa_window", m_compute_params.swa_window); + } create_1d_input("n_seq_active", m_compute_params.n_seq_active); create_1d_input("seq_active_start", m_compute_params.seq_active_start); create_1d_input("seq_active_end", m_compute_params.seq_active_start + m_compute_params.n_seq_active); @@ -1278,7 +1471,7 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor void GgmlOvDecoder::dump_cgraph(const ggml_cgraph * cgraph, std::string & filename) { std::ofstream file(filename); if (!file.is_open()) { - std::cerr << "Failed to open file" << std::endl; + std::cerr << "Failed to open file" << '\n'; return; } @@ -1384,11 +1577,11 @@ void print_tensor_address_map(const ggml_cgraph * cgraph) { } } for (const auto & pair : address_map) { - std::cout << "Address: " << pair.first << std::endl; + std::cout << "Address: " << pair.first << '\n'; for (const auto & name : pair.second) { std::cout << name << " ; "; } - std::cout << std::endl << std::endl; + std::cout << "\n\n"; } } @@ -2030,7 +2223,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { std::cout << ", "; } } - std::cout << "]" << std::endl; + std::cout << "]" << '\n'; // print the src name & shape with the dynamic dim for debugging for (int j = 0; j < GGML_MAX_SRC; j++) { ggml_tensor * src = node->src[j]; @@ -2049,9 +2242,9 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { std::cout << ", "; } } - std::cout << "]" << std::endl; + std::cout << "]" << '\n'; } - std::cout << std::endl; + std::cout << '\n'; } } } diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 74cb7385029a..056e39e87170 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -21,18 +21,28 @@ struct ModelParams { int ctx_per_seq_swa = -1; int n_seq = 1; int n_heads_kv = -1; + // Per-layer KV head count. gemma-4 12B interleaves 8 x 256 sliding layers with 1 x 512 + // full-attention layers, so no single scalar describes every layer. Keyed by layer, not by + // layer TYPE, because the SWA classification depends on the context size (extents tie at a + // small -c) while the head count does not. + std::map<int, int> n_heads_kv_per_layer; int head_size = -1; int state_size = -1; // for SSM molels, eg qwen35 - int32_t rope_params[15]; + int32_t rope_params[16]; bool mixed_rope_params = false; + bool is_cacheless_attn = false; std::vector<int> swa_layers; + // The sliding-window mask tensor, identified in compute_llm_params() by grouping attention + // layers on the mask they consume. Only used to tell the two masks apart when naming OV + // parameters -- both carry the same tensor name. Null when the graph has a single mask. + const ggml_tensor * swa_mask = nullptr; std::vector<std::string> kv_names; size_t kv_buffer_ctx_id = 0; bool same_rope_params(const ModelParams & other) const { return mixed_rope_params == other.mixed_rope_params && - memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0; + memcmp(rope_params, other.rope_params, sizeof(int32_t) * 16) == 0; } bool can_reuse_dynamically(const ModelParams & other) const { return same_rope_params(other); } @@ -48,6 +58,11 @@ struct ComputeParams { int attention_size = -1; int attention_size_swa = -1; int attention_size_static = -1; // encoder/cross-attn KV fill level (whisper) + // Sliding window width, read back from the band of ggml's own SWA mask. ggml never passes + // n_swa down to a backend, but fill_mask() bakes it into the mask contents, so the widest + // unmasked row recovers it. Shorter than n_swa while the sequence is still short, which is + // harmless: every causal pair is inside the window then anyway. + int swa_window = -1; int input_len = -1; int token_len_per_seq = -1; int past_kv_len = -1; @@ -96,8 +111,15 @@ struct ComputeParams { // models use a fixed end-anchored offset in the translator. }; +// defined below; declared here because GgmlOvDecoder uses it inline +std::optional<int> extract_layer_from_name(const std::string & name); + +// detects the MoE expert-plane-sum ADD chain (see definition); used by supports_op too +bool is_moe_expert_sum_add(const ggml_tensor * node); + class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { public: + static std::string get_tensor_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor); struct NodeInfo { ggml_tensor * node; std::string node_name; @@ -250,6 +272,21 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { m_model_params.swa_layers.end(); } + // KV head count for one layer. Sliding and full layers can differ (gemma-4 12B), so callers + // that reinterpret a KV buffer must use this and not the model-level n_heads_kv. + int get_n_heads_kv_for_layer(int layer) const { + auto it = m_model_params.n_heads_kv_per_layer.find(layer); + return it != m_model_params.n_heads_kv_per_layer.end() ? it->second : m_model_params.n_heads_kv; + } + + // Same, for a KV cache tensor: its layer comes from the leaf name (cache_k_l<N>). + int get_n_heads_kv_for_tensor(const ggml_tensor * kv_tensor) const { + if (auto layer = extract_layer_from_name(std::string(kv_tensor->name)); layer.has_value()) { + return get_n_heads_kv_for_layer(layer.value()); + } + return m_model_params.n_heads_kv; + } + int get_past_kv_len() const { return m_compute_params.past_kv_len; } int get_input_len() const { return m_compute_params.input_len; } @@ -317,35 +354,41 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { void update_io(ggml_cgraph * cgraph); - inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE; } - inline static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_ROPE && tensor == op->src[1]; } // IMROPE packs 4 stacked position planes (t/h/w/e) into inp_pos, each of length // n_tokens; other modes carry a single position per token. - inline static int get_inp_pos_n_planes(const ggml_tensor * op) { + static int get_inp_pos_n_planes(const ggml_tensor * op) { return op->op_params[2] == GGML_ROPE_TYPE_IMROPE ? 4 : 1; } - inline static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) { return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM; } - inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) || (op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]); } - inline static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_mean(const ggml_tensor * tensor, const ggml_tensor * op) { + return op->op == GGML_OP_MUL_MAT && tensor == op->src[1] && tensor->op == GGML_OP_NONE && + (tensor->flags & GGML_TENSOR_FLAG_INPUT) && tensor->type == GGML_TYPE_F32 && + op->src[0] != nullptr && op->src[0]->op != GGML_OP_NONE; + } + + static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_ROPE && tensor == op->src[2]; } // also returns true for cache_s and cache_r in SSM/DeltaNet models - inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) { if (tensor == nullptr) { return false; } @@ -353,17 +396,28 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { (op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor); } - inline static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_conv_state_writeback(const ggml_tensor * node) { + return node->op == GGML_OP_CPY && node->view_src != nullptr && is_kvcache(node->view_src, nullptr) && + node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr && + node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr && + node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src; + } + + static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_SET_ROWS && op->src[1] == tensor; } - inline static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { + bool is_swa_mask(const ggml_tensor * tensor) const { + return m_model_params.swa_mask != nullptr && tensor == m_model_params.swa_mask; + } + + static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE && op->src[1]->op == GGML_OP_NONE; } // the state permutation index input used in SSM/DeltaNet models (inp->s_copy in llama-graph.cpp) - inline static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY; } @@ -375,8 +429,22 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { if (is_inp_emb(tensor, op)) { return "embd"; } - if (is_stateful() && is_inp_mask(tensor, op)) { - return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa"; + if (is_inp_mask(tensor, op)) { + // Give the two attention masks distinct OV parameter names. + // + // An interleaved-SWA model builds one full-attention mask and one sliding-window mask, + // but build_attn_inp_kq_mask() names them identically, so keying a parameter off + // tensor->name alone makes the second mask OVERWRITE the first in m_model_inputs: both + // attention types then read a single parameter, and the windowed layers silently run + // against an unbanded mask. Disambiguate using the SWA layer set computed in + // compute_llm_params(), which classifies by mask tensor identity rather than by name. + // + // When no SWA layer was found there is only one mask in play, so the plain name is + // correct and no _swa parameter is created. + if (m_model_params.swa_layers.empty()) { + return "self_kq_mask"; + } + return is_swa_mask(tensor) ? "self_kq_mask_swa" : "self_kq_mask"; } return tensor->name; } @@ -413,5 +481,3 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { }; void print_tensor_address_map(const ggml_cgraph * cgraph); - -std::optional<int> extract_layer_from_name(const std::string & name); diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 36dfa4d9471b..216e3b8a69e2 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -31,6 +31,7 @@ void ggml_openvino_device_config::init() { // String values (use ggml_openvino_getenv_str) "GGML_OPENVINO_DEVICE", "GGML_OPENVINO_CACHE_DIR", + "GGML_OPENVINO_SPILL_DIR", "GGML_OPENVINO_DEBUG_NODE", "GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR", "GGML_OPENVINO_NPU_COMPILE_CONFIG", @@ -56,6 +57,11 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_RELEASE_WEIGHTS", "GGML_OPENVINO_REDUCE_COMPILE_MEM", "GGML_OPENVINO_LOG_UNSUPPORTED_OPS", + "GGML_OPENVINO_LOG_SWA_LAYERS", + "GGML_OPENVINO_NATIVE_SOFTPLUS", + "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS", + "GGML_OPENVINO_REQUANT_KQUANT", + "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT", }; for (const char * const & env_var : env_var_names) { @@ -263,9 +269,81 @@ std::optional<ExtraQuantType> ggml_openvino_get_requant_type(const ggml_tensor * if (ggml_openvino_is_npu()) { return ExtraQuantType::Q4_0_128; } + // By default Q6_K/Q5_K are requantized to Q8_0_C, which *inflates* 6- and 5-bit weights to 8 + // while the rest of the model stays at 4 bits, and Q4_K keeps its native group-32 layout + // (an f16 scale plus an f16 zero point per 32 weights = 0.125 B/weight of metadata). + // Decode of a large model is bandwidth-bound, so both cost throughput. + // + // GGML_OPENVINO_REQUANT_KQUANT selects a 4-bit target instead. Names are + // q4_<sym|asym><group>[_all]: <sym|asym> says whether a per-group zero point is kept, <group> + // is the group size, and the _all suffix sends Q4_K down the same path (without it only + // Q6_K/Q5_K are touched): + // q4_sym128 Q6_K/Q5_K -> Q4_0_128 (u4, group 128, symmetric) + // q4_sym128_all and Q4_K too -- drops Q4_K's per-32 zero point, which costs some accuracy + // q4_asym64_all Q6_K/Q5_K and Q4_K -> Q4_1_64 (u4, group 64, asymmetric) -- most of the + // metadata saving while keeping a real zero point + // native no requantization at all (keep Q6_K/Q5_K as they are) + // + // The asymmetric target is only offered in its _all form: leaving Q4_K at its native group 32 + // while Q6_K/Q5_K move to group 64 gives the Q/K/V projections different group counts, and the + // GPU plugin's FullyConnectedHorizontalFusion concatenates their scale constants, which then + // fails shape inference. Requantizing all three keeps the group size uniform. + const char * rq = ggml_openvino_getenv_str("GGML_OPENVINO_REQUANT_KQUANT"); + auto is_opt = [rq](const char * name) { + return rq && strcmp(rq, name) == 0; + }; + const bool sym128 = is_opt("q4_sym128"); + const bool sym128_all = is_opt("q4_sym128_all"); + const bool asym64_all = is_opt("q4_asym64_all"); + + if (tensor->type == GGML_TYPE_Q4_K) { + if (sym128_all) { + return ExtraQuantType::Q4_0_128; + } + if (asym64_all) { + return ExtraQuantType::Q4_1_64; + } + } + // MoE expert weights (3D, ne[2] = n_expert) stored as Q5_1/Q8_0 are the expert-side + // equivalent of Q6_K/Q5_K: kept at 8 bits by default while the rest of the model is at 4 + // (gemma-4 26B-A4B keeps its down projection there). Send them to 4 bits under the same + // option, at group 64 rather than 128: the down expert has k=704, which 64 divides + // (704/64 = 11) and 128 does not. + if (tensor->ne[2] > 1 && (tensor->type == GGML_TYPE_Q5_1 || tensor->type == GGML_TYPE_Q8_0)) { + if (sym128 || sym128_all) { + return ExtraQuantType::Q4_0_64; + } + if (asym64_all) { + return ExtraQuantType::Q4_1_64; + } + // TODO: temporary workaround for a known OpenVINO GPU-plugin bug -- remove once the + // plugin computes grouped 8-bit GatherMatmulCompressed correctly. This costs accuracy + // (5/8-bit -> 4-bit) on any model it applies to, so it must not outlive the bug. + // + // On GPU these would otherwise stay in their native *grouped 8-bit* layout, which the GPU + // plugin's GatherMatmulCompressed computes incorrectly -- gemma-4 26B-A4B (whose down + // projection is Q5_1) produces garbage, while the same graph is correct on CPU. It is + // specific to grouped 8 bit: the gate/up experts are grouped u4 *with* a zero point and + // are fine, and Qwen3.5 / granite are fine because their Q5_K/Q6_K down projections + // already requantize to per-channel Q8_0_C (grouped=0). Sending these to grouped 4 bit + // avoids the broken layout and restores correct output. + // Opt out with GGML_OPENVINO_REQUANT_KQUANT=native. + if (ggml_openvino_get_device_name() == "GPU" && !is_opt("native")) { + return ExtraQuantType::Q4_0_64; + } + } switch (tensor->type) { case GGML_TYPE_Q6_K: case GGML_TYPE_Q5_K: + if (sym128 || sym128_all) { + return ExtraQuantType::Q4_0_128; + } + if (asym64_all) { + return ExtraQuantType::Q4_1_64; + } + if (is_opt("native")) { + return std::nullopt; + } return ExtraQuantType::Q8_0_C; default: return std::nullopt; @@ -331,6 +409,16 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.weights_per_block = 128; layout.is_symmetric = true; break; + case ExtraQuantType::Q4_1_64: + layout.is_u4 = true; + layout.weights_per_block = 64; + layout.is_symmetric = false; + break; + case ExtraQuantType::Q4_0_64: + layout.is_u4 = true; + layout.weights_per_block = 64; + layout.is_symmetric = true; + break; case ExtraQuantType::Q4_0_C: layout.is_u4 = true; layout.weights_per_block = tensor->ne[0]; @@ -384,10 +472,6 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten switch (tensor->type) { case GGML_TYPE_MXFP4: - layout.is_u4 = true; - layout.is_symmetric = true; - break; - case GGML_TYPE_Q4_0: layout.is_u4 = true; layout.is_symmetric = true; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index 0916b416258f..9d827d969452 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -15,7 +15,10 @@ #include <string> // ExtraQuantType enum - defines requantization target formats -enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q8_0_C, Q8_0_32 }; +// Q4_1_64: u4, group 64, *true* asymmetric (per-group scale and zero point). Note that +// Q4_0_128/Q4_0_C are symmetric despite taking the unsigned branch of quantize_q4_0 -- that branch +// pins zp to 8 with d = max/-8, which is algebraically symmetric. +enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q4_0_64, Q8_0_C, Q8_0_32, Q4_1_64 }; ov::Core & ov_singleton_core(); diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 4b1789713d1d..02c5962238a7 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -10,7 +10,10 @@ #include "ggml.h" #include <atomic> +#include <cerrno> +#include <climits> #include <cstdint> +#include <cstdio> #include <cstdlib> #include <cstring> #include <memory> @@ -25,7 +28,7 @@ #include <string> #include <vector> -#if defined(_WIN32) +#ifdef _WIN32 # define WIN32_LEAN_AND_MEAN # ifndef NOMINMAX # define NOMINMAX @@ -53,6 +56,7 @@ // - CPU repack buffer: tensor->extra stores tensor_traits with repacked data // ===================================================== +namespace { // Buffer context that manages per-tensor allocations (no contiguous buffer for weights) struct ggml_backend_openvino_buffer_context { int device; @@ -64,6 +68,11 @@ struct ggml_backend_openvino_buffer_context { size_t size; bool is_remote; + // Set when the buffer is a file-backed spill mapping (GGML_OPENVINO_SPILL_DIR); it must be + // munmap'd rather than freed. + void * spill_mapping = nullptr; + size_t spill_size = 0; + // Wrapping of the buffer std::shared_ptr<ov::Tensor> ov_buffer; @@ -98,10 +107,56 @@ struct ggml_backend_openvino_buffer_context { data = usm_tensor.get(); ov_buffer = std::make_shared<ov::intel_gpu::ocl::USMTensor>(std::move(usm_tensor)); } else { - data = ggml_aligned_malloc(size); - GGML_ASSERT(data); - memset(data, 0, size); - ov_buffer = std::make_shared<ov::Tensor>(ov::element::u8, ov::Shape{size}, data); +#ifndef _WIN32 + if (const char * spill_dir = ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) { + // Disk-backed weight buffer: back the repacked weights with a temp file via MAP_SHARED + // instead of anonymous memory. Anonymous pages can only be evicted to swap, so the + // repacked buffer stays pinned alongside the mmap'd source and both are resident at once + // -- that double residency is the load-time peak. File-backed pages are reclaimable: the + // kernel can write them back and drop them under pressure, then re-read on demand, so RSS + // becomes a working set rather than the whole buffer. The file is unlinked immediately, + // so it disappears when the process exits. + // + // The directory must be real storage. Pointing this at a tmpfs mount (/tmp on many + // systems) backs the "spill" with RAM and makes matters worse. + char path[PATH_MAX]; + snprintf(path, sizeof(path), "%s/ggml-ov-weights-%d-XXXXXX", spill_dir, (int) getpid()); + int fd = mkstemp(path); + if (fd < 0) { + GGML_LOG_ERROR("%s: mkstemp(%s) failed: %s\n", __func__, path, strerror(errno)); + return; + } + unlink(path); // anonymous-but-file-backed: freed on process exit + if (ftruncate(fd, (off_t) size) != 0) { + GGML_LOG_ERROR("%s: ftruncate(%zu) failed: %s\n", __func__, size, strerror(errno)); + close(fd); + return; + } + void * m = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + close(fd); // the mapping keeps the file alive + if (m == MAP_FAILED) { + GGML_LOG_ERROR("%s: mmap(%zu) failed: %s\n", __func__, size, strerror(errno)); + return; + } + data = m; + spill_mapping = m; + spill_size = size; + GGML_LOG_INFO("%s: weight buffer spilled to %s (%zu MB, file-backed)\n", __func__, spill_dir, + size / 1024 / 1024); + ov_buffer = std::make_shared<ov::Tensor>(ov::element::u8, ov::Shape{size}, data); + } else +#endif + { +#ifdef _WIN32 + if (ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) { + GGML_LOG_WARN("%s: GGML_OPENVINO_SPILL_DIR is not supported on Windows, ignoring\n", __func__); + } +#endif + data = ggml_aligned_malloc(size); + GGML_ASSERT(data); + memset(data, 0, size); + ov_buffer = std::make_shared<ov::Tensor>(ov::element::u8, ov::Shape{size}, data); + } } if (data == nullptr) { @@ -124,6 +179,11 @@ struct ggml_backend_openvino_buffer_context { delete pair.second; } tensor_extras.clear(); +#ifndef _WIN32 + if (spill_mapping != nullptr) { + munmap(spill_mapping, spill_size); + } else +#endif if (!is_remote && data != nullptr) { ggml_aligned_free(data, size); } @@ -135,6 +195,7 @@ struct ggml_backend_openvino_buffer_type_context { int device; std::string name; }; +} // namespace // ===================================================== // Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS) @@ -194,14 +255,16 @@ void ggml_openvino_release_weight_buffers() { for (const auto & b : reg.buffers) { // Align down/up to page boundaries so madvise only drops whole pages // fully owned by this buffer. - const long page = sysconf(_SC_PAGESIZE); - uintptr_t start = reinterpret_cast<uintptr_t>(b.first); - uintptr_t end = start + b.second; - uintptr_t astart = (start + page - 1) & ~(uintptr_t) (page - 1); - uintptr_t aend = end & ~(uintptr_t) (page - 1); - if (aend > astart) { - if (madvise(reinterpret_cast<void *>(astart), aend - astart, MADV_DONTNEED) == 0) { - total += aend - astart; + const size_t page = (size_t) sysconf(_SC_PAGESIZE); + const uintptr_t ustart = reinterpret_cast<uintptr_t>(b.first); + const size_t offset_to_page = (page - (ustart & (page - 1))) & (page - 1); + if (b.second > offset_to_page) { + const size_t aligned_len = (b.second - offset_to_page) & ~(page - 1); + if (aligned_len > 0) { + char * astart = static_cast<char *>(b.first) + offset_to_page; + if (madvise(astart, aligned_len, MADV_DONTNEED) == 0) { + total += aligned_len; + } } } } @@ -611,9 +674,7 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_buffer_type(in static const char * ggml_backend_openvino_host_buffer_type_get_name(ggml_backend_buffer_type_t buft) { ggml_backend_openvino_buffer_type_context * ctx = (ggml_backend_openvino_buffer_type_context *) buft->context; - static std::string name; - name = ctx->name + "_HOST"; - return name.c_str(); + return ctx->name.c_str(); } static bool ggml_backend_openvino_host_buffer_type_is_host(ggml_backend_buffer_type_t buft) { @@ -646,7 +707,7 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_host_buffer_ty for (int i = 0; i < device_count; i++) { buffer_type_contexts[i].device = i; - buffer_type_contexts[i].name = std::string(GGML_OPENVINO_NAME) + std::to_string(i); + buffer_type_contexts[i].name = std::string(GGML_OPENVINO_NAME) + std::to_string(i) + "_HOST"; buffer_types[i] = ggml_backend_buffer_type{ /* .iface = */ ggml_backend_openvino_host_buffer_type_interface, @@ -711,13 +772,16 @@ static void ggml_backend_openvino_free(ggml_backend_t backend) { if (ctx->runtime_context) { auto r_ctx = std::static_pointer_cast<ov_runtime_context>(ctx->runtime_context); - if (--r_ctx->backend_count == 0) { + auto cache = r_ctx->compiled_cache; + r_ctx->clear_caches(); + std::lock_guard<std::mutex> cache_lock(cache->mutex); + if (--cache->backend_count == 0) { // If host weight buffers were released (GGML_OPENVINO_RELEASE_WEIGHTS), the // dropped pages can never be repopulated, so a recompile is impossible. Keep // the compiled-model cache alive across backend teardown so the next context // reuses it instead of recompiling against zeroed weights. if (!ggml_openvino_weight_buffers_released()) { - r_ctx->clear_caches(); + cache->graphs.clear(); } } } @@ -766,12 +830,14 @@ static ggml_guid_t ggml_backend_openvino_guid(void) { } static std::shared_ptr<ov_runtime_context> get_ov_runtime_context_ptr() { - static std::shared_ptr<ov_runtime_context> r_ctx = [] { - auto ctx = std::make_shared<ov_runtime_context>(); - ctx->device = ggml_openvino_get_device_name(); - ctx->stateful = is_stateful_enabled() && !ggml_openvino_is_npu(); - return ctx; - }(); + // Share compiled models, but give every backend its own requests and KV state. + static auto cache = std::make_shared<ov_compiled_model_cache>(); + auto r_ctx = std::make_shared<ov_runtime_context>(); + r_ctx->device = ggml_openvino_get_device_name(); + r_ctx->stateful = is_stateful_enabled() && !ggml_openvino_is_npu(); + r_ctx->compiled_cache = cache; + std::lock_guard<std::mutex> cache_lock(cache->mutex); + ++cache->backend_count; return r_ctx; } @@ -795,9 +861,6 @@ GGML_BACKEND_API ggml_backend_t ggml_backend_openvino_init(int device) { return nullptr; } - std::shared_ptr<ov_runtime_context> r_ctx = std::static_pointer_cast<ov_runtime_context>(ctx->runtime_context); - r_ctx->backend_count++; - ggml_backend_t openvino_backend = new ggml_backend{ /* .guid = */ ggml_backend_openvino_guid(), /* .interface = */ ggml_backend_openvino_interface, @@ -812,11 +875,13 @@ GGML_BACKEND_API bool ggml_backend_is_openvino(ggml_backend_t backend) { return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_openvino_guid()); } +namespace { struct ggml_backend_openvino_device_context { int device; std::string name; std::string description; }; +} static const char * ggml_backend_openvino_device_get_name(ggml_backend_dev_t dev) { ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context; @@ -928,6 +993,10 @@ static bool is_supported_flash_attn_pattern(const ggml_tensor * op) { if (src->src[0] == nullptr || src->src[0]->view_src != nullptr) { return false; } + } else if (src->op == GGML_OP_CPY) { + if (src->src[0] == nullptr || src->src[0]->op != GGML_OP_PERMUTE || src->src[0]->src[0] == nullptr) { + return false; + } } else { return false; } @@ -995,7 +1064,7 @@ static bool cpy_output_view_is_supported(const ggml_tensor * op) { return false; } - return ggml_nbytes(op) == 0 || ggml_is_contiguous(op); + return ggml_nbytes(op) == 0 || ggml_is_contiguous(op) || GgmlOvDecoder::is_conv_state_writeback(op); } static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) { @@ -1091,6 +1160,10 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { if (op->ne[3] != 1) { return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"}; } + if (op->op == GGML_OP_GET_ROWS && ggml_is_quantized(op->src[0]->type) && + op->src[0]->view_src != nullptr && op->src[0]->view_offs != 0) { + return {false, "GET_ROWS with a nonzero quantized src0 view offset is not supported"}; + } if (op->op == GGML_OP_GET_ROWS && ggml_openvino_get_device_name() == "GPU" && op->src[0]->type == GGML_TYPE_BF16) { return {false, "GET_ROWS with BF16 src0 is not supported on GPU"}; @@ -1119,6 +1192,10 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { if (op->src[1]->op == GGML_OP_PERMUTE) { return {false, "ADD/MUL/SUB with PERMUTE src1 is not supported"}; } + // >8-expert MoE ReduceSum drifts past the 1e-7 tolerance (f32 order vs CPU); intermittent. + if (op->op == GGML_OP_ADD && is_moe_expert_sum_add(op) && op->src[1]->src[0]->ne[1] > 8) { + return {false, "MoE expert-plane sum with more than 8 experts is not supported"}; + } for (int i = 0; i < 4; i++) { if (op->src[0]->ne[i] != op->src[1]->ne[i] && (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1)) { return {false, "ADD/MUL/SUB with incompatible broadcast shapes: src0->ne[" + std::to_string(i) + "]=" + @@ -1203,8 +1280,11 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { break; } case GGML_OP_CPY: { - if (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16) { - return {false, "CPY with BF16 src type is not supported"}; + if (op->src[0]->type != GGML_TYPE_BF16 && op->src[1]->type == GGML_TYPE_BF16) { + return {false, "CPY with BF16 src[1] type is not supported"}; + } + if (ggml_openvino_get_device_name() == "NPU" && (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16)) { + return {false, "CPY with BF16 is not supported is not supported on NPU"}; } // CPY to a quantized destination (e.g. f32 -> q4_0) is numerically unstable with OpenVINO backend. if (ggml_is_quantized(op->type)) { @@ -1234,6 +1314,10 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { op->src[0]->ne[0] == 256 && op->src[1]->ne[0] == 256) { return {false, "MUL_MAT quantized benchmark test case on GPU is not supported"}; } + if (ggml_openvino_get_device_name() == "GPU" && op->type == GGML_TYPE_F32 && op->ne[0] == 1 && op->ne[1] == 1 && + (op->src[0]->buffer == nullptr || op->src[0]->buffer->usage != GGML_BACKEND_BUFFER_USAGE_WEIGHTS)) { + return {false, "MUL_MAT scalar dot product with non-weight src[0] on GPU is not supported"}; + } if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) { return {false, "MUL_MAT with incompatible broadcast on ne[3]: src0->ne[3]=" + std::to_string(op->src[0]->ne[3]) + ", src1->ne[3]=" + std::to_string(op->src[1]->ne[3])}; @@ -1250,14 +1334,23 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { return {false, "MUL_MAT_ID with single-expert or empty ne[2] <= 1 (ne[2]=" + std::to_string(op->src[0]->ne[2]) + ") is not supported"}; } - if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_BF16) { - return {false, "MUL_MAT_ID with BF16 weights on GPU is not supported"}; + if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && !ggml_is_quantized(op->src[0]->type)) { + return {false, "MUL_MAT_ID with non-quantized weights on GPU is not supported"}; + } + // The GPU plugin's GatherMatmul returns wrong values for the layouts test-backend-ops + // produces: it builds a rank-4 input layout ([n_used, n_tokens, k, 1]) instead of rank 3 + // and the kernel misreads it, silently returning garbage (NMSE ~86) rather than asserting. + // The same graph is correct on the CPU plugin, and correct on GPU for every real model, + // which always feeds experts from a bound tensor buffer. Standalone op-test tensors have + // no buffer at all, so use that to exclude them and let the scheduler run them on CPU. + if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[0]->buffer == nullptr) { + return {false, "MUL_MAT_ID with unbound expert tensors on GPU is not supported"}; } - // GPU MUL_MAT_ID uses a Gather+MatMul fallback because the GPU plugin rejects internal - // GatherMatmul for these test shapes. Skip cases that would materialize a large selected - // expert-weight temporary. - if (ggml_openvino_get_device_name() == "GPU" && mul_mat_id_requires_large_tmp(op)) { - return {false, "MUL_MAT_ID requires large temporary on GPU"}; + // Only MXFP4 still needs the large-temporary guard; every other quantized type goes + // through GatherMatmul, which never materializes the selected expert weights. + if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4 && + mul_mat_id_requires_large_tmp(op)) { + return {false, "MUL_MAT_ID with MXFP4 weights requires large temporary on GPU"}; } break; } @@ -1265,36 +1358,39 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { const int32_t * op_params = op->op_params; const int n_dims = op_params[1]; const int mode = op_params[2]; - if (op_params[15] != 0) { - // FIXME: support ggml_rope_set_offset - return {false, "ggml_rope_set_offset is not supported"}; - } + const int64_t n_offs = op_params[15]; if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX && mode != GGML_ROPE_TYPE_IMROPE) { return {false, "ROPE with mode " + std::to_string(mode) + " is not supported"}; } + if (n_offs < 0 || (n_offs % 2) != 0) { + return {false, "ROPE with invalid n_offs=" + std::to_string(n_offs)}; + } const int64_t head_dim = op->src[0]->ne[0]; const int64_t rope_dims = n_dims == 0 ? head_dim : n_dims; - if (rope_dims <= 0 || rope_dims > head_dim || (rope_dims % 2) != 0) { - return {false, "ROPE with n_dims=" + std::to_string(n_dims) + ", head_dim=" + std::to_string(head_dim) + " is not supported"}; + if (rope_dims <= 0 || rope_dims + n_offs > head_dim || (rope_dims % 2) != 0) { + return {false, "ROPE with n_dims=" + std::to_string(n_dims) + ", n_offs=" + std::to_string(n_offs) + + ", head_dim=" + std::to_string(head_dim) + " is not supported"}; } if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) { return {false, "ROPE with type " + std::string(ggml_type_name(op->type)) + " is not supported"}; } - if (op->src[0]->op == GGML_OP_VIEW) { - const struct ggml_tensor * view = op->src[0]; - const struct ggml_tensor * view_src = view->view_src; - if (view_src->ne[1] != view->ne[1] || view_src->ne[2] != view->ne[2] || view_src->ne[3] != view->ne[3]) { - return {false, "ROPE with view_src->ne [" + std::to_string(view_src->ne[1]) + ", " + - std::to_string(view_src->ne[2]) + ", " + std::to_string(view_src->ne[3]) + - "] != view->ne [" + std::to_string(view->ne[1]) + ", " + - std::to_string(view->ne[2]) + ", " + std::to_string(view->ne[3]) + - "] is not supported"}; - } + if (op->view_src != nullptr && !ggml_is_contiguous(op->src[0])) { + return {false, "ROPE on VIEW / non-contiguous input is not supported"}; + } + if (op->src[0]->ne[3] > 1) { + // translate_rope's cos/sin tables cover one sequence only; ne[3] > 1 fails to broadcast. + return {false, "ROPE with multiple sequences (ne[3]=" + std::to_string(op->src[0]->ne[3]) + + ") is not supported"}; } + float freq_scale; + float ext_factor; + float attn_factor; + memcpy(&freq_scale, op_params + 6, sizeof(float)); + memcpy(&ext_factor, op_params + 7, sizeof(float)); + memcpy(&attn_factor, op_params + 8, sizeof(float)); if (mode == GGML_ROPE_TYPE_IMROPE && - (op->src[2] != 0 || ((const float *) op_params)[6] != 1 || ((const float *) op_params)[7] != 0 || - ((const float *) op_params)[8] != 1)) { - return {false, "IMROPE with freq_factors, freq_scale, ext_factor, and attn_factor is not supported"}; + (op->src[2] != nullptr || freq_scale != 1.0f || ext_factor != 0.0f || attn_factor != 1.0f)) { + return {false, "IMROPE with freq_factors, freq_scale, ext_factor, or attn_factor is not supported"}; } break; } @@ -1493,9 +1589,11 @@ static const struct ggml_backend_device_i ggml_backend_openvino_device_interface /* .event_synchronize = */ NULL, }; +namespace { struct ggml_backend_openvino_reg_context { std::vector<ggml_backend_dev_t> devices; }; +} static const char * ggml_backend_openvino_reg_get_name(ggml_backend_reg_t reg) { return GGML_OPENVINO_NAME; diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index 120db01e17cd..824d24478290 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -34,6 +34,15 @@ #include <string> #include <vector> +// From <openvino>/src/common/transformations/include/transformations/utils/utils.hpp +namespace ov::op::util { +// From <openvino>/src/common/transformations/include/transformations/utils/utils.hpp +bool get_single_value(const std::shared_ptr<ov::op::v0::Constant> & const_node, + float & value, + bool check_value_range = true); +} // namespace ov::op::util + +namespace { void unpack_32_4(const uint8_t * data, uint8_t * dst) { std::fill_n(dst, 16, 0); for (int j = 0; j < 16; ++j) { @@ -48,11 +57,11 @@ void unpack_32_4(const uint8_t * data, uint8_t * dst) { } } -static constexpr size_t MXFP4_BLOCK_SIZE = 32; -static constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2; -static constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE; +constexpr size_t MXFP4_BLOCK_SIZE = 32; +constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2; +constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE; -static void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) { +void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) { for (int j = 0; j < static_cast<int>(MXFP4_BLOCK_QS_SIZE); j += 2) { const uint8_t v0 = data[j] & 0x0F; const uint8_t v1 = (data[j + 1] & 0x0F) << 4; @@ -419,7 +428,7 @@ void extract_q6_k_data(const ggml_tensor * tensor, } } -static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { +inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { if (j < 4) { *d = q[j] & 63; *m = q[j + 4] & 63; @@ -514,9 +523,9 @@ void extract_q5_k_data(const ggml_tensor * tensor, ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight, ov::Tensor & scales, ov::Tensor & zp, - size_t group_size, - bool use_bias, - bool for_gather_matmul) { + size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, + bool use_bias = false, + bool for_gather_matmul = false) { ov::Shape orig_shape = weight.get_shape(); bool is_signed = (weight.get_element_type() == ov::element::i8); // Symmetric: signed weights, no ZP @@ -611,13 +620,24 @@ ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight, return std::make_shared<ov::op::v0::Convert>(result, ov::element::f32); } +// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights +// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as +// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in +// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see +// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul +// directly from the resulting f16 dequant chain. +// +// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp +// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point); +// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not +// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern. // See make_int8_weights for the meaning of for_gather_matmul. ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight, ov::Tensor & scales, ov::Tensor & zp, - size_t group_size, - bool use_bias, - bool for_gather_matmul) { + size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, + bool use_bias = false, + bool for_gather_matmul = false) { ov::Shape orig_weight_shape = weight.get_shape(); bool is_signed = (weight.get_element_type() == ov::element::i4); // Symmetric: signed weights, no ZP @@ -746,13 +766,262 @@ ov::Output<ov::Node> make_mxfp4_moe_packed_weights(ov::Tensor & weight) { return weights_node; } +void quantize_q4_0(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk) { + assert(k % qk == 0); + const int nb = k / qk; + + auto * weights = static_cast<uint8_t *>(weights_arr.data()); + auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>(); + bool is_symmetric = (weights_arr.get_element_type() == ov::element::i4); // Signed i4 path + + if (!is_symmetric) { + auto * zp = static_cast<uint8_t *>(zp_arr.data()); + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + float max = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + if (amax < fabsf(v)) { + amax = fabsf(v); + max = v; + } + } + const float d = max / -8; + if (d == 0) { + scales[i] = ov::float16(1.0f); + if (i % 2 == 0) { + zp[i / 2] = 8; + } else { + zp[i / 2] |= (8 << 4); + } + memset(weights + i * qk / 2, 8 | (8 << 4), qk / 2); + continue; + } + const float id = 1.0f / d; + scales[i] = ov::float16(d); + if (i % 2 == 0) { + zp[i / 2] = 8; + } else { + zp[i / 2] |= (8 << 4); + } + for (int j = 0; j < qk / 2; ++j) { + const float x0 = x[i * qk + 2 * j] * id; + const float x1 = x[i * qk + 2 * j + 1] * id; + const uint8_t xi0 = MIN(15, (int8_t) (x0 + 8.5f)); + const uint8_t xi1 = MIN(15, (int8_t) (x1 + 8.5f)); + weights[i * qk / 2 + j] = xi0 | (xi1 << 4); + } + } + } else { + // Symmetric: produce signed i4 values in [-8, 7] + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + float max = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + if (amax < fabsf(v)) { + amax = fabsf(v); + max = v; + } + } + const float d = max / -8; + if (d == 0) { + scales[i] = ov::float16(1.0f); + // i4 value 0 packed: 0x00 + memset(weights + i * qk / 2, 0, qk / 2); + continue; + } + const float id = 1.0f / d; + scales[i] = ov::float16(d); + for (int j = 0; j < qk / 2; ++j) { + const float x0 = x[i * qk + 2 * j] * id; + const float x1 = x[i * qk + 2 * j + 1] * id; + // Signed i4: range [-8, 7]. Quantize as round(x*id), then pack as 4-bit two's complement. + int8_t si0 = (int8_t) std::max(-8, std::min(7, (int) roundf(x0))); + int8_t si1 = (int8_t) std::max(-8, std::min(7, (int) roundf(x1))); + weights[i * qk / 2 + j] = (si0 & 0x0F) | ((si1 & 0x0F) << 4); + } + } + } +} + +// Asymmetric u4 quantization with a per-group scale and zero point. +// +// Unlike quantize_q4_0's unsigned branch, which pins the zero point to 8 and is therefore +// symmetric, this keeps a real per-group zero point, so a group whose values are not centred on +// zero does not waste half its range. +void quantize_q4_1_asym(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk) { + assert(k % qk == 0); + const int nb = k / qk; + + auto * weights = static_cast<uint8_t *>(weights_arr.data()); + auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>(); + auto * zp = static_cast<uint8_t *>(zp_arr.data()); + + // u4 zero points are packed two per byte, low nibble first, indexed by group -- the same + // convention as the unsigned branch of quantize_q4_0. + auto store_zp = [zp](int i, uint8_t v) { + if (i % 2 == 0) { + zp[i / 2] = v & 0x0F; + } else { + zp[i / 2] |= (uint8_t) ((v & 0x0F) << 4); + } + }; + + for (int i = 0; i < nb; i++) { + float vmin = x[i * qk]; + float vmax = x[i * qk]; + for (int j = 1; j < qk; j++) { + const float v = x[i * qk + j]; + vmin = std::min(vmin, v); + vmax = std::max(vmax, v); + } + // Include 0 in the range so an all-positive or all-negative group still represents zero + // exactly -- these are weights, so an exact zero matters. + vmin = std::min(vmin, 0.0f); + vmax = std::max(vmax, 0.0f); + + const float d = (vmax - vmin) / 15.0f; + if (d == 0.0f) { + scales[i] = ov::float16(1.0f); + store_zp(i, 0); + memset(weights + i * qk / 2, 0, qk / 2); + continue; + } + const float id = 1.0f / d; + + // The zero point is itself a 4-bit integer, so round it and dequantize as (q - zq) * d. + const int zq = std::max(0, std::min(15, (int) lroundf(-vmin * id))); + scales[i] = ov::float16(d); + store_zp(i, (uint8_t) zq); + + for (int j = 0; j < qk / 2; ++j) { + const float x0 = x[i * qk + 2 * j] * id; + const float x1 = x[i * qk + 2 * j + 1] * id; + const uint8_t q0 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x0) + zq)); + const uint8_t q1 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x1) + zq)); + weights[i * qk / 2 + j] = (uint8_t) (q0 | (q1 << 4)); + } + } +} + +void quantize_q8_0(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk, + int64_t block_offset = 0) { + assert(k % qk == 0); + const int nb = k / qk; + + // block_offset lets a caller quantize a chunk of blocks into the right place in the + // output buffers (used for streaming requant). x points at this chunk's first block; + // outputs are advanced by block_offset blocks. Q8 has one scale/zp per block (no + // nibble packing), so any block boundary is safe. + auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk; + auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset; + bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8); // Signed i8 path + + if (!is_symmetric) { + auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset; + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + amax = std::max(amax, fabsf(v)); + } + const float d = amax / 127.0f; + const float id = d ? 1.0f / d : 0.0f; + scales[i] = ov::float16(d); + zp[i] = 128; + for (int j = 0; j < qk; ++j) { + const float x0 = x[i * qk + j] * id; + const int8_t xi0 = roundf(x0); + weights[i * qk + j] = (uint8_t) (xi0 + 128); + } + } + } else { + // Symmetric: store signed int8 values directly + auto * signed_weights = reinterpret_cast<int8_t *>(weights); + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + amax = std::max(amax, fabsf(v)); + } + const float d = amax / 127.0f; + const float id = d ? 1.0f / d : 0.0f; + scales[i] = ov::float16(d); + for (int j = 0; j < qk; ++j) { + const float x0 = x[i * qk + j] * id; + signed_weights[i * qk + j] = (int8_t) roundf(x0); + } + } + } +} + +void quantize_q8_1(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk, + int64_t block_offset = 0) { + assert(k % qk == 0); + const int nb = k / qk; + + // See quantize_q8_0: block_offset places this chunk's output at the right block. + auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk; + auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset; + auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset; + for (int i = 0; i < nb; i++) { + float min = std::numeric_limits<float>::max(); + float max = std::numeric_limits<float>::lowest(); + + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + min = std::min(v, min); + max = std::max(v, max); + } + + const float d = (max - min) / ((1 << 8) - 1); + const float id = d ? 1.0f / d : 0.0f; + scales[i] = ov::float16(d); + // zp = -min / scale (Q8_1 is asymmetric) + zp[i] = (d != 0.0f) ? (uint8_t) std::round(-min / d) : 0; + + for (int j = 0; j < qk; ++j) { + const float x0 = (x[i * qk + j] - min) * id; + const uint8_t xi0 = roundf(x0); + weights[i * qk + j] = xi0; + } + } +} + // Extract quantized weights from tensor and create weight subgraph +// If weights/scales/zp are provided (non-empty), uses them as output buffers +// Otherwise allocates new ov::Tensors internally +// Returns the weight node (make_int4_weights or make_int8_weights result) std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor, - const void * data, + const void * data, // Source data pointer (may differ from tensor->data) ov::Tensor & weights, ov::Tensor & scales, ov::Tensor & zp, - bool use_bias) { + // Use an exact f16 zero point (vs. a rounded integer one); always + // used for for_gather_matmul (3D MoE expert) weights regardless of + // this flag, and also settable explicitly for test-backend-ops. + bool use_bias = false) { // Create a temporary tensor for extraction functions that read from tensor->data ggml_tensor temp_tensor = *tensor; temp_tensor.data = const_cast<void *>(data); @@ -837,9 +1106,11 @@ std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor, return result; } -// Requantize weights to target format, writing to provided buffers +// Requantize weights from tensor to target format, writing to provided buffers +// For F16 target, only weights buffer is used (scales/zp ignored) +// Returns the weight node std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor, - const void * data, + const void * data, // Source data pointer ExtraQuantType requant_type, int64_t block_size, ov::Tensor & weights, @@ -851,7 +1122,8 @@ std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor, const auto * type_traits = ggml_get_type_traits(tensor->type); const size_t src_row_bytes = ggml_row_size(tensor->type, ne0); - bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128); + bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128 || + requant_type == ExtraQuantType::Q4_0_64 || requant_type == ExtraQuantType::Q4_1_64); // Streaming dequant (opt-in via GGML_OPENVINO_REDUCE_COMPILE_MEM or // GGML_OPENVINO_MEMORY_OPTIMIZE): instead of @@ -879,7 +1151,9 @@ std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor, result->set_friendly_name(tensor->name); return result; } - if (is_u4) { + if (requant_type == ExtraQuantType::Q4_1_64) { + quantize_q4_1_asym(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } else if (is_u4) { quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); } else if (requant_type == ExtraQuantType::Q8_1_C) { quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size); @@ -930,6 +1204,7 @@ std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor, result->set_friendly_name(tensor->name); return result; } +} // namespace OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, void * output_base_ptr, bool use_bias) { GGML_ASSERT(tensor != nullptr); @@ -1027,7 +1302,9 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo } else { result.weights = ov::Tensor(ov::element::f16, node_shape); } - ov::Tensor dummy_scales, dummy_zp; // Not used for F16 + // Not used for F16: + ov::Tensor dummy_scales; + ov::Tensor dummy_zp; result.weight_node = requantize_to_buffers(tensor, data, ExtraQuantType::F16, 0, result.weights, dummy_scales, dummy_zp); return result; @@ -1036,10 +1313,14 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo // Quantized path (normal extraction or quantized requant) // Create weight/scale/zp tensors - shared between both paths // For symmetric quantization, use signed types (i4/i8) and no ZP tensor - ov::element::Type weight_type = tensor->type == GGML_TYPE_MXFP4 ? - ov::element::f4e2m1 : - (layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) : - (layout.is_u4 ? ov::element::u4 : ov::element::u8)); + ov::element::Type weight_type; + if (tensor->type == GGML_TYPE_MXFP4) { + weight_type = ov::element::f4e2m1; + } else if (layout.is_symmetric) { + weight_type = layout.is_u4 ? ov::element::i4 : ov::element::i8; + } else { + weight_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; + } ov::Shape scale_shape = node_shape; scale_shape.back() /= layout.weights_per_block; @@ -1057,28 +1338,25 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo scale_shape.back() /= layout.weights_per_block; } + const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; + ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; + if (zp_is_f16) { + zp_type = ov::element::f16; + } + if (output_base_ptr) { uint8_t * buf_base = static_cast<uint8_t *>(output_base_ptr); result.weights = ov::Tensor(weight_type, node_shape, buf_base + layout.weights_offset); - const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset); if (!layout.is_symmetric) { - ov::element::Type zp_type = - zp_is_f16 ? ov::element::f16 : (layout.is_u4 ? ov::element::u4 : ov::element::u8); result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset); } // else: result.zp remains default-constructed (empty) for symmetric } else { result.weights = ov::Tensor(weight_type, node_shape); - const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; result.scales = ov::Tensor(scale_type, scale_shape); if (!layout.is_symmetric) { - if (zp_is_f16) { - result.zp = ov::Tensor(ov::element::f16, scale_shape); - } else { - ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; - result.zp = ov::Tensor(zp_type, scale_shape); - } + result.zp = ov::Tensor(zp_type, scale_shape); } // else: result.zp remains default-constructed (empty) for symmetric } @@ -1093,181 +1371,3 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo return result; } - -void quantize_q4_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk) { - assert(k % qk == 0); - const int nb = k / qk; - - auto * weights = static_cast<uint8_t *>(weights_arr.data()); - auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>(); - bool is_symmetric = (weights_arr.get_element_type() == ov::element::i4); // Signed i4 path - - if (!is_symmetric) { - auto * zp = static_cast<uint8_t *>(zp_arr.data()); - for (int i = 0; i < nb; i++) { - float amax = 0.0f; - float max = 0.0f; - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - if (amax < fabsf(v)) { - amax = fabsf(v); - max = v; - } - } - const float d = max / -8; - if (d == 0) { - scales[i] = ov::float16(1.0f); - if (i % 2 == 0) { - zp[i / 2] = 8; - } else { - zp[i / 2] |= (8 << 4); - } - memset(weights + i * qk / 2, 8 | (8 << 4), qk / 2); - continue; - } - const float id = 1.0f / d; - scales[i] = ov::float16(d); - if (i % 2 == 0) { - zp[i / 2] = 8; - } else { - zp[i / 2] |= (8 << 4); - } - for (int j = 0; j < qk / 2; ++j) { - const float x0 = x[i * qk + 2 * j] * id; - const float x1 = x[i * qk + 2 * j + 1] * id; - const uint8_t xi0 = MIN(15, (int8_t) (x0 + 8.5f)); - const uint8_t xi1 = MIN(15, (int8_t) (x1 + 8.5f)); - weights[i * qk / 2 + j] = xi0 | (xi1 << 4); - } - } - } else { - // Symmetric: produce signed i4 values in [-8, 7] - for (int i = 0; i < nb; i++) { - float amax = 0.0f; - float max = 0.0f; - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - if (amax < fabsf(v)) { - amax = fabsf(v); - max = v; - } - } - const float d = max / -8; - if (d == 0) { - scales[i] = ov::float16(1.0f); - // i4 value 0 packed: 0x00 - memset(weights + i * qk / 2, 0, qk / 2); - continue; - } - const float id = 1.0f / d; - scales[i] = ov::float16(d); - for (int j = 0; j < qk / 2; ++j) { - const float x0 = x[i * qk + 2 * j] * id; - const float x1 = x[i * qk + 2 * j + 1] * id; - // Signed i4: range [-8, 7]. Quantize as round(x*id), then pack as 4-bit two's complement. - int8_t si0 = (int8_t) std::max(-8, std::min(7, (int) roundf(x0))); - int8_t si1 = (int8_t) std::max(-8, std::min(7, (int) roundf(x1))); - weights[i * qk / 2 + j] = (si0 & 0x0F) | ((si1 & 0x0F) << 4); - } - } - } -} - -void quantize_q8_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk, - int64_t block_offset) { - assert(k % qk == 0); - const int nb = k / qk; - - // block_offset lets a caller quantize a chunk of blocks into the right place in the - // output buffers (used for streaming requant). x points at this chunk's first block; - // outputs are advanced by block_offset blocks. Q8 has one scale/zp per block (no - // nibble packing), so any block boundary is safe. - auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk; - auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset; - bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8); // Signed i8 path - - if (!is_symmetric) { - auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset; - for (int i = 0; i < nb; i++) { - float amax = 0.0f; - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - amax = std::max(amax, fabsf(v)); - } - const float d = amax / 127.0f; - const float id = d ? 1.0f / d : 0.0f; - scales[i] = ov::float16(d); - zp[i] = 128; - for (int j = 0; j < qk; ++j) { - const float x0 = x[i * qk + j] * id; - const int8_t xi0 = roundf(x0); - weights[i * qk + j] = (uint8_t) (xi0 + 128); - } - } - } else { - // Symmetric: store signed int8 values directly - auto * signed_weights = reinterpret_cast<int8_t *>(weights); - for (int i = 0; i < nb; i++) { - float amax = 0.0f; - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - amax = std::max(amax, fabsf(v)); - } - const float d = amax / 127.0f; - const float id = d ? 1.0f / d : 0.0f; - scales[i] = ov::float16(d); - for (int j = 0; j < qk; ++j) { - const float x0 = x[i * qk + j] * id; - signed_weights[i * qk + j] = (int8_t) roundf(x0); - } - } - } -} - -void quantize_q8_1(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk, - int64_t block_offset) { - assert(k % qk == 0); - const int nb = k / qk; - - // See quantize_q8_0: block_offset places this chunk's output at the right block. - auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk; - auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset; - auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset; - for (int i = 0; i < nb; i++) { - float min = std::numeric_limits<float>::max(); - float max = std::numeric_limits<float>::lowest(); - - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - min = std::min(v, min); - max = std::max(v, max); - } - - const float d = (max - min) / ((1 << 8) - 1); - const float id = d ? 1.0f / d : 0.0f; - scales[i] = ov::float16(d); - // zp = -min / scale (Q8_1 is asymmetric) - zp[i] = (d != 0.0f) ? (uint8_t) std::round(-min / d) : 0; - - for (int j = 0; j < qk; ++j) { - const float x0 = (x[i * qk + j] - min) * id; - const uint8_t xi0 = roundf(x0); - weights[i * qk + j] = xi0; - } - } -} diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h index e247255a7f77..04fe0218a672 100644 --- a/ggml/src/ggml-openvino/ggml-quants.h +++ b/ggml/src/ggml-openvino/ggml-quants.h @@ -2,112 +2,12 @@ #include "ggml-openvino-extra.h" // For ExtraQuantType #include "ggml.h" -#include <cstdint> -#include <openvino/op/constant.hpp> #include <openvino/core/node_output.hpp> +#include <openvino/op/constant.hpp> #include <openvino/runtime/tensor.hpp> -void unpack_32_4(const uint8_t * data, uint8_t * dst); - -void extract_q4_0_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr); - -void extract_q4_1_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - bool use_bias = false); - -void extract_q5_1_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - bool use_bias = false); - -void extract_q8_0_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr); - -void unpack_256_4(const uint8_t * data, uint8_t * dst); - -void extract_q4_k_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - bool use_bias = false); - -void extract_q5_k_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - bool use_bias = false); - -void extract_q6_k_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr); - -void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr); - static constexpr size_t GGML_QUANTIZATION_GROUP_SIZE = 32; -// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights -// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as -// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in -// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see -// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul -// directly from the resulting f16 dequant chain. -// -// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp -// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point); -// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not -// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern. -ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight, - ov::Tensor & scales, - ov::Tensor & zp, - size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, - bool use_bias = false, - bool for_gather_matmul = false); - -ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight, - ov::Tensor & scales, - ov::Tensor & zp, - size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, - bool use_bias = false, - bool for_gather_matmul = false); - -ov::Output<ov::Node> make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales); - -ov::Output<ov::Node> make_mxfp4_moe_packed_weights(ov::Tensor & weight); - -// Extract quantized weights from tensor and create weight subgraph -// If weights/scales/zp are provided (non-empty), uses them as output buffers -// Otherwise allocates new ov::Tensors internally -// Returns the weight node (make_int4_weights or make_int8_weights result) -std::shared_ptr<ov::Node> extract_quantized_weights( - const ggml_tensor * tensor, - const void * data, // Source data pointer (may differ from tensor->data) - ov::Tensor & weights, - ov::Tensor & scales, - ov::Tensor & zp, - bool use_bias = false); // Use an exact f16 zero point (vs. a rounded integer one); always - // used for for_gather_matmul (3D MoE expert) weights regardless of - // this flag, and also settable explicitly for test-backend-ops. - -// Requantize weights from tensor to target format, writing to provided buffers -// For F16 target, only weights buffer is used (scales/zp ignored) -// Returns the weight node -std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor, - const void * data, // Source data pointer - ExtraQuantType requant_type, - int64_t block_size, - ov::Tensor & weights, - ov::Tensor & scales, - ov::Tensor & zp); - inline const char * extra_quant_type_name(ExtraQuantType t) { switch (t) { case ExtraQuantType::F16: @@ -122,6 +22,10 @@ inline const char * extra_quant_type_name(ExtraQuantType t) { return "Q8_0_32"; case ExtraQuantType::Q8_1_C: return "Q8_1_C"; + case ExtraQuantType::Q4_0_64: + return "Q4_0_64"; + case ExtraQuantType::Q4_1_64: + return "Q4_1_64"; default: return "unknown"; } @@ -152,35 +56,3 @@ OvWeight process_weight_tensor( // always used for for_gather_matmul (3D MoE expert) weights // regardless of this flag, and also settable explicitly for // test-backend-ops. - -void quantize_q4_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk); -void quantize_q8_1(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk, - int64_t block_offset = 0); -void quantize_q8_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk, - int64_t block_offset = 0); - -namespace ov { -namespace op { -namespace util { -// From <openvino>/src/common/transformations/include/transformations/utils/utils.hpp -bool get_single_value(const std::shared_ptr<ov::op::v0::Constant> & const_node, - float & value, - bool check_value_range = true); -} // namespace util -} // namespace op -} // namespace ov diff --git a/ggml/src/ggml-openvino/model-cache.cpp b/ggml/src/ggml-openvino/model-cache.cpp index 3fc7028d88bc..3725fbd2252e 100644 --- a/ggml/src/ggml-openvino/model-cache.cpp +++ b/ggml/src/ggml-openvino/model-cache.cpp @@ -237,7 +237,8 @@ bool ggml_openvino_model_cache_verify_manifest(const std::string & path, if (!f.is_open()) { return false; } - std::string tag, val; + std::string tag; + std::string val; // header: fingerprint if (!(f >> tag >> val) || tag != "fingerprint" || val != hex64(fingerprint)) { return false; diff --git a/ggml/src/ggml-openvino/openvino/frontend.cpp b/ggml/src/ggml-openvino/openvino/frontend.cpp index c2ba14e66e6e..88de86feacae 100644 --- a/ggml/src/ggml-openvino/openvino/frontend.cpp +++ b/ggml/src/ggml-openvino/openvino/frontend.cpp @@ -3,6 +3,7 @@ #include "input_model.h" #include "op_table.h" #include "translate_session.h" +#include <openvino/core/type.hpp> namespace ov { namespace frontend { @@ -11,7 +12,7 @@ namespace ggml { FrontEnd::FrontEnd() {} std::shared_ptr<Model> FrontEnd::convert(const InputModel::Ptr & model, bool naive) { - auto ggml_model = std::dynamic_pointer_cast<ggml::InputModel>(model); + auto ggml_model = ov::as_type_ptr<ggml::InputModel>(model); FRONT_END_GENERAL_CHECK(ggml_model, "Invalid input model"); std::shared_ptr<Model> converted_model; const auto & supported_ops = get_supported_ops(); diff --git a/ggml/src/ggml-openvino/openvino/frontend.h b/ggml/src/ggml-openvino/openvino/frontend.h index 72134a3e8cf2..4e301d32e07a 100644 --- a/ggml/src/ggml-openvino/openvino/frontend.h +++ b/ggml/src/ggml-openvino/openvino/frontend.h @@ -12,7 +12,6 @@ namespace ggml { class FrontEnd { public: - using Ptr = std::shared_ptr<FrontEnd>; FrontEnd(); static std::shared_ptr<Model> convert(const InputModel::Ptr & model, bool naive = false); diff --git a/ggml/src/ggml-openvino/openvino/node_context.h b/ggml/src/ggml-openvino/openvino/node_context.h index 2e2756037703..f1ea0e4f0eac 100644 --- a/ggml/src/ggml-openvino/openvino/node_context.h +++ b/ggml/src/ggml-openvino/openvino/node_context.h @@ -143,6 +143,10 @@ class NodeContext : public frontend::NodeContext { bool has_input(const std::string & name) const { return m_tensor_map->find(name) != m_tensor_map->end(); } + void put_shared(const std::string & name, const Output<Node> & value) const { + m_tensor_map->insert({name, value}); + } + const std::string & get_name() const override { return m_decoder->get_op_name(m_node_idx); } ov::Any get_attribute_as_any(const std::string & name) const override { return m_decoder->get_attribute(name); } diff --git a/ggml/src/ggml-openvino/openvino/op/add.cpp b/ggml/src/ggml-openvino/openvino/op/add.cpp index c43eb67f8d28..a45520d92e5b 100644 --- a/ggml/src/ggml-openvino/openvino/op/add.cpp +++ b/ggml/src/ggml-openvino/openvino/op/add.cpp @@ -5,6 +5,7 @@ #include <memory> #include <openvino/op/add.hpp> #include <openvino/op/constant.hpp> +#include <openvino/op/convert.hpp> #include <openvino/op/reduce_sum.hpp> #include <openvino/op/unsqueeze.hpp> @@ -35,7 +36,20 @@ OutputVector translate_add(const NodeContext & context) { auto input_0 = process_view_input_new(context, 0); auto input_1 = process_view_input_new(context, 1); - auto res = std::make_shared<ov::op::v1::Add>(input_0, input_1); + // opset1::Add needs matching types (e.g. fused ADD_ADD mixes f16/f32); add in f32, cast once. + auto output_type = context.get_output_type(); + if (input_0.get_element_type() != input_1.get_element_type()) { + if (input_0.get_element_type() != ov::element::f32) { + input_0 = std::make_shared<ov::op::v0::Convert>(input_0, ov::element::f32); + } + if (input_1.get_element_type() != ov::element::f32) { + input_1 = std::make_shared<ov::op::v0::Convert>(input_1, ov::element::f32); + } + } + ov::Output<ov::Node> res = std::make_shared<ov::op::v1::Add>(input_0, input_1); + if (res.get_element_type() != output_type) { + res = std::make_shared<ov::op::v0::Convert>(res, output_type); + } return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/add_id.cpp b/ggml/src/ggml-openvino/openvino/op/add_id.cpp index e54d700d421a..79bdbe87731e 100644 --- a/ggml/src/ggml-openvino/openvino/op/add_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/add_id.cpp @@ -20,7 +20,7 @@ namespace op { static ov::Output<ov::Node> reshape_add_id_input_to_2d(const ov::Output<ov::Node> & input, const ov::PartialShape & input_shape, const std::vector<int> & dims) { - const auto actual_shape = input.get_partial_shape(); + const auto & actual_shape = input.get_partial_shape(); if (actual_shape.rank().is_static() && actual_shape.rank().get_length() == 2) { return input; } diff --git a/ggml/src/ggml-openvino/openvino/op/cont.cpp b/ggml/src/ggml-openvino/openvino/op/cont.cpp index 1d6cc6721260..9888f6b93fd3 100644 --- a/ggml/src/ggml-openvino/openvino/op/cont.cpp +++ b/ggml/src/ggml-openvino/openvino/op/cont.cpp @@ -3,12 +3,9 @@ #include "../op_table.h" #include "../utils.h" -#include <climits> -#include <cstdint> #include <memory> #include <openvino/op/reshape.hpp> #include <openvino/op/slice.hpp> -#include <vector> namespace ov { namespace frontend { diff --git a/ggml/src/ggml-openvino/openvino/op/diag.cpp b/ggml/src/ggml-openvino/openvino/op/diag.cpp index dacea2f05b4a..05e064892e17 100644 --- a/ggml/src/ggml-openvino/openvino/op/diag.cpp +++ b/ggml/src/ggml-openvino/openvino/op/diag.cpp @@ -3,11 +3,8 @@ #include "../utils.h" #include <openvino/op/constant.hpp> -#include <openvino/op/equal.hpp> +#include <openvino/op/eye.hpp> #include <openvino/op/multiply.hpp> -#include <openvino/op/range.hpp> -#include <openvino/op/reshape.hpp> -#include <openvino/op/select.hpp> namespace ov { namespace frontend { @@ -23,31 +20,13 @@ namespace op { OutputVector translate_diag(const NodeContext & context) { num_inputs_check(context, 1, 1); - auto x = context.get_input(0); // OV shape: [ne3, ne2, 1, ne0] + auto x = process_view_input_new(context, 0); // OV shape: [ne3, ne2, 1, ne0] - auto out_shape = context.get_output_shape().to_shape(); - int64_t n = static_cast<int64_t>(out_shape[3]); // ne0 + auto n = get_dimensions(x.get_node_shared_ptr(), {3}); + auto zero_diag = ov::op::v0::Constant::create(ov::element::i64, {}, {0}); - // Build index range [0, 1, ..., n-1] - auto start = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(0)}); - auto stop = ov::op::v0::Constant::create(ov::element::i64, {}, {n}); - auto step = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(1)}); - auto range = std::make_shared<ov::op::v4::Range>(start, stop, step, ov::element::i64); - - // col_idx shape [1, 1, 1, n] - auto col_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, 1, n}); - auto col_idx = std::make_shared<ov::op::v1::Reshape>(range, col_shape, false); - - // row_idx shape [1, 1, n, 1] - auto row_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, n, 1}); - auto row_idx = std::make_shared<ov::op::v1::Reshape>(range, row_shape, false); - - // mask: true where col == row (diagonal) - auto mask = std::make_shared<ov::op::v1::Equal>(col_idx, row_idx); - - // Broadcast input from [ne3, ne2, 1, ne0] to [ne3, ne2, ne0, ne0] via select - auto zero = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f}); - auto res = std::make_shared<ov::op::v1::Select>(mask, x, zero); + auto eye = std::make_shared<ov::op::v9::Eye>(n, n, zero_diag, x.get_element_type()); + auto res = std::make_shared<ov::op::v1::Multiply>(x, eye); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/div.cpp b/ggml/src/ggml-openvino/openvino/op/div.cpp index 11dd9decec7a..2089ffd4c819 100644 --- a/ggml/src/ggml-openvino/openvino/op/div.cpp +++ b/ggml/src/ggml-openvino/openvino/op/div.cpp @@ -4,12 +4,14 @@ #include "ggml.h" #include <memory> +#include <openvino/core/type.hpp> #include <openvino/op/constant.hpp> #include <openvino/op/convert.hpp> #include <openvino/op/divide.hpp> #include <openvino/op/multiply.hpp> #include <openvino/op/shape_of.hpp> #include <openvino/op/sigmoid.hpp> +#include <openvino/op/swish.hpp> #include <openvino/op/tile.hpp> #include <openvino/op/util/precision_sensitive_attribute.hpp> #include <vector> @@ -33,22 +35,12 @@ bool is_silu_div_pattern(const ov::Output<ov::Node> & numerator, return false; } - auto mul = std::dynamic_pointer_cast<ov::op::v1::Multiply>(numerator.get_node_shared_ptr()); - if (!mul) { - return false; - } - const auto denom_node = denominator.get_node_shared_ptr(); - const auto mul_input_0 = mul->input_value(0).get_node_shared_ptr(); - const auto mul_input_1 = mul->input_value(1).get_node_shared_ptr(); - auto sigmoid = std::dynamic_pointer_cast<ov::op::v0::Sigmoid>(mul_input_1); - if (mul_input_0 == denom_node && sigmoid && sigmoid->input_value(0).get_node_shared_ptr() == denom_node) { - return true; + if (auto swish = ov::as_type_ptr<ov::op::v4::Swish>(numerator.get_node_shared_ptr())) { + return swish->input_value(0).get_node_shared_ptr() == denom_node; } - - sigmoid = std::dynamic_pointer_cast<ov::op::v0::Sigmoid>(mul_input_0); - return mul_input_1 == denom_node && sigmoid && sigmoid->input_value(0).get_node_shared_ptr() == denom_node; + return false; } ov::Output<ov::Node> repeat_input_to_match(const NodeContext & context, diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp index 06547f3d2968..b06d01dcace0 100644 --- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp @@ -195,7 +195,9 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { auto tile_kv = [&](int64_t n_heads, int64_t n_heads_kv, int64_t hs, ov::Output<Node> kv) { int64_t f = n_heads / n_heads_kv; if (f > 1 && n_heads_kv > 1) { - ov::Output<ov::Node> kv_broadcast_shape, kv_unsqueezed, new_kv_shape; + ov::Output<ov::Node> kv_broadcast_shape; + ov::Output<ov::Node> kv_unsqueezed; + ov::Output<ov::Node> new_kv_shape; auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2}); kv_unsqueezed = std::make_shared<ov::op::v0::Unsqueeze>(kv, unsqueeze_axes); diff --git a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp index 07eeb3c8fd6d..8d07c90bfec1 100644 --- a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +++ b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp @@ -196,7 +196,7 @@ static OutputVector translate_gated_delta_net_ref(const NodeContext & context) { } // Merge batch and head dims: [B*H_v, T, S_v] - auto merge_bh = [&](ov::Output<ov::Node> x, int64_t last_dim) { + auto merge_bh = [&](const ov::Output<ov::Node> & x, int64_t last_dim) { auto shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{B * H_v, T, last_dim}); return std::make_shared<ov::op::v1::Reshape>(x, shape, false); }; diff --git a/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp b/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp index c6d64aed43aa..385d75f5ffd6 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp @@ -6,8 +6,8 @@ #include <openvino/core/node_output.hpp> #include <openvino/op/constant.hpp> #include <openvino/op/multiply.hpp> -#include <openvino/op/sigmoid.hpp> #include <openvino/op/slice.hpp> +#include <openvino/op/swish.hpp> namespace ov { namespace frontend { @@ -50,9 +50,7 @@ OutputVector translate_glu_geglu_quick(const NodeContext & context) { // Create the constant in the same type as src0 to avoid f16/f32 mismatch. auto input_type = src0.get_element_type(); auto coef = ov::op::v0::Constant::create(input_type, ov::Shape{}, {1.702f}); - auto scaled = std::make_shared<ov::op::v1::Multiply>(src0, coef); - auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(scaled); - auto gated = std::make_shared<ov::op::v1::Multiply>(src0, sigmoid); + auto gated = std::make_shared<ov::op::v4::Swish>(src0, coef); auto res = std::make_shared<ov::op::v1::Multiply>(gated, src1); return rename_outputs_with_suffix({res}, context.get_name()); diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp index d81fc53b5d02..7eea81d96e59 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp @@ -9,9 +9,10 @@ #include <openvino/op/add.hpp> #include <openvino/op/clamp.hpp> #include <openvino/op/constant.hpp> +#include <openvino/op/convert.hpp> #include <openvino/op/multiply.hpp> -#include <openvino/op/sigmoid.hpp> #include <openvino/op/slice.hpp> +#include <openvino/op/swish.hpp> namespace ov { namespace frontend { @@ -61,8 +62,7 @@ static std::pair<ov::Output<ov::Node>, ov::Output<ov::Node>> get_glu_inputs(cons OutputVector translate_glu_swiglu(const NodeContext & context) { auto [src0, src1] = get_glu_inputs(context); - auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(src0); - auto silu = std::make_shared<ov::op::v1::Multiply>(src0, sigmoid); + auto silu = std::make_shared<ov::op::v4::Swish>(src0); auto res = std::make_shared<ov::op::v1::Multiply>(silu, src1); return rename_outputs_with_suffix({res}, context.get_name()); @@ -77,9 +77,7 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) { auto gate = std::make_shared<ov::op::v0::Clamp>(src0, -std::numeric_limits<float>::infinity(), limit); auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha}); - auto scaled_gate = std::make_shared<ov::op::v1::Multiply>(gate, alpha_const); - auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(scaled_gate); - auto out_glu = std::make_shared<ov::op::v1::Multiply>(gate, sigmoid); + auto out_glu = std::make_shared<ov::op::v4::Swish>(gate, alpha_const); auto up = std::make_shared<ov::op::v0::Clamp>(src1, -limit, limit); auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f}); @@ -95,11 +93,22 @@ OutputVector translate_glu_swiglu_clamp(const NodeContext & context) { const int32_t * params = context.get_output_op_params(); const float limit = reinterpret_cast<const float *>(params)[3]; + // Compute in f32: f16 Swish/Clamp rounding drifts past the 1e-7 test tolerance. + auto output_type = context.get_output_type(); + if (src0.get_element_type() != ov::element::f32) { + src0 = std::make_shared<ov::op::v0::Convert>(src0, ov::element::f32); + } + if (src1.get_element_type() != ov::element::f32) { + src1 = std::make_shared<ov::op::v0::Convert>(src1, ov::element::f32); + } + auto gate = std::make_shared<ov::op::v0::Clamp>(src0, -std::numeric_limits<float>::infinity(), limit); - auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(gate); - auto silu = std::make_shared<ov::op::v1::Multiply>(gate, sigmoid); + auto silu = std::make_shared<ov::op::v4::Swish>(gate); auto up = std::make_shared<ov::op::v0::Clamp>(src1, -limit, limit); - auto res = std::make_shared<ov::op::v1::Multiply>(silu, up); + ov::Output<ov::Node> res = std::make_shared<ov::op::v1::Multiply>(silu, up); + if (res.get_element_type() != output_type) { + res = std::make_shared<ov::op::v0::Convert>(res, output_type); + } return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/im2col.cpp b/ggml/src/ggml-openvino/openvino/op/im2col.cpp index 856e97f79d86..08b53f260d63 100644 --- a/ggml/src/ggml-openvino/openvino/op/im2col.cpp +++ b/ggml/src/ggml-openvino/openvino/op/im2col.cpp @@ -1,7 +1,6 @@ #include "../node_context.h" #include "../op_table.h" #include "../utils.h" -#include "ggml-impl.h" #include <cstddef> #include <memory> diff --git a/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp b/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp new file mode 100644 index 000000000000..07e94c690152 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp @@ -0,0 +1,90 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// Local mirror of OpenVINO's internal ov::op::internal::MOE and MOECompressed ops. +// +// The class bodies are provided by the linked libopenvino.so; only the declarations are +// needed here so the backend can construct the node directly (same approach as +// GatherMatmul and GatedDeltaNet). The class layout must stay in sync with +// openvino/src/core/dev_api/openvino/op/moe.hpp +// openvino/src/common/transformations/include/ov_ops/moe_compressed.hpp +// +// \note MOE op classes are under development and subject to change. + +#pragma once + +#include <optional> + +#include "openvino/core/type/element_type.hpp" +#include "openvino/op/op.hpp" + +namespace ov::op::internal { + +class OPENVINO_API MOE : public ov::op::Op { +public: + OPENVINO_OP("MOE") + + MOE() = default; + + MOE(const OutputVector & args) : Op(args) {} + + enum class Expert_type { GEMM2_BIAS_SWIGLU_CLAMP, GEMM3_SWIGLU }; + + enum class Activation_type { SWIGLU, GEGLU_TANH, GEGLU_ERF }; + + struct Config { + Expert_type expert_type{ Expert_type::GEMM2_BIAS_SWIGLU_CLAMP }; + float expert_alpha{ 0.0f }; + float expert_beta{ 1.0f }; + size_t gate_idx{ 0 }; + Activation_type activation_type{ Activation_type::SWIGLU }; + }; + + MOE(const OutputVector & args, const Config & config); + + const Config & get_config() const; + void set_config(const Config & config); + + bool visit_attributes(AttributeVisitor & visitor) override; + void validate_and_infer_types() override; + std::shared_ptr<Node> clone_with_new_inputs(const OutputVector & new_args) const override; + +private: + Config m_config; +}; + +class OPENVINO_API MOECompressed : public MOE { +public: + OPENVINO_OP("MOECompressed", "", ov::op::internal::MOE) + + MOECompressed() = default; + + struct Config : public MOE::Config { + size_t hidden_size = 0; + size_t inter_size = 0; + size_t num_expert = 0; + size_t num_shared_expert = 0; + size_t top_k = 0; + // numeric_limits<size_t>::max() means per_channel compression (single group) + size_t group_size = 0; + bool has_batch_dim = false; + bool has_zp = false; + ov::element::Type out_type = ov::element::dynamic; + std::optional<float> scale_factor; + }; + + MOECompressed(const OutputVector & args, const Config & config); + + const Config & get_config() const { return m_config; } + + void set_scale_factor(float scale_factor) { m_config.scale_factor = scale_factor; } + + bool visit_attributes(AttributeVisitor & visitor) override; + void validate_and_infer_types() override; + std::shared_ptr<Node> clone_with_new_inputs(const OutputVector & new_args) const override; + +protected: + Config m_config; +}; + +} // namespace ov::op::internal diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp index f1b28c85d401..a336924e14fa 100644 --- a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp @@ -42,7 +42,7 @@ ov::Output<ov::Node> slice_axis(const ov::Output<ov::Node> & input, int64_t axis ov::Output<ov::Node> static_shape_dims_or_shapeof(const ov::Output<ov::Node> & input, const std::vector<int> & dims) { - const auto partial_shape = input.get_partial_shape(); + const auto & partial_shape = input.get_partial_shape(); if (partial_shape.is_static()) { std::vector<int64_t> values; values.reserve(dims.size()); @@ -56,54 +56,6 @@ ov::Output<ov::Node> static_shape_dims_or_shapeof(const ov::Output<ov::Node> & i return get_dimensions(shape, dims); } -ov::Output<ov::Node> translate_mul_mat_id_gather_matmul_fallback(const NodeContext & context, - ov::Output<ov::Node> expert_weights, - ov::Output<ov::Node> activations, - ov::Output<ov::Node> ids) { - auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); - ov::Output<ov::Node> selected_weights = std::make_shared<ov::op::v8::Gather>(expert_weights, ids, gather_axis); - - const auto output_type = context.get_output_type(); - if (selected_weights.get_element_type() != ov::element::f32) { - selected_weights = std::make_shared<ov::op::v0::Convert>(selected_weights, ov::element::f32); - } - if (activations.get_element_type() != ov::element::f32) { - activations = std::make_shared<ov::op::v0::Convert>(activations, ov::element::f32); - } - - auto activations_shape = std::make_shared<ov::op::v3::ShapeOf>(activations, ov::element::i64); - auto ids_shape = std::make_shared<ov::op::v3::ShapeOf>(ids, ov::element::i64); - ov::Output<ov::Node> acts_target_dims = std::make_shared<ov::op::v0::Concat>( - ov::OutputVector{ - get_dimensions(activations_shape, {0}), - get_dimensions(ids_shape, {1}), - get_dimensions(activations_shape, {2}), - }, - 0); - ov::Output<ov::Node> acts_broadcasted = - std::make_shared<ov::op::v3::Broadcast>(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL); - - auto activations_expanded = std::make_shared<ov::op::v0::Unsqueeze>(acts_broadcasted, const_i64({2})); - ov::Output<ov::Node> result = - std::make_shared<ov::op::v0::MatMul>(activations_expanded, selected_weights, false, true); - - auto output_shape = context.get_output_shape(); - FRONT_END_OP_CONVERSION_CHECK(output_shape.rank().is_static() && output_shape.rank().get_length() == 4, - "Unexpected MUL_MAT_ID output rank"); - FRONT_END_OP_CONVERSION_CHECK(output_shape[3].is_static(), "Expected static row dimension for MUL_MAT_ID output"); - - auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3].get_length()}); - auto result_target_dims = std::make_shared<ov::op::v0::Concat>( - ov::OutputVector{batch_dim, get_dimensions(ids_shape, {0, 1}), row_dim}, 0); - result = std::make_shared<ov::op::v1::Reshape>(result, result_target_dims, false); - - if (result.get_element_type() != output_type) { - result = std::make_shared<ov::op::v0::Convert>(result, output_type); - } - return result; -} - ov::Output<ov::Node> translate_mul_mat_id_mxfp4_packed(const NodeContext & context, ov::Output<ov::Node> expert_weights, ov::Output<ov::Node> activations, @@ -229,7 +181,6 @@ OutputVector translate_mul_mat_id(const NodeContext & context) { auto expert_weights_rank = expert_weights.get_partial_shape().rank(); FRONT_END_OP_CONVERSION_CHECK(expert_weights_rank.is_static(), "Expected static rank for MUL_MAT_ID expert weights"); - const bool use_gpu_fallback = ggml_openvino_get_device_name() == "GPU"; if (expert_weights_rank.get_length() == 4) { auto expert_weights_shape_3d = static_shape_dims_or_shapeof(expert_weights, {1, 2, 3}); expert_weights = std::make_shared<ov::op::v1::Reshape>(expert_weights, expert_weights_shape_3d, false); @@ -246,14 +197,9 @@ OutputVector translate_mul_mat_id(const NodeContext & context) { } const auto output_type = context.get_output_type(); - if (activations.get_element_type() != ov::element::f32) { - activations = std::make_shared<ov::op::v0::Convert>(activations, ov::element::f32); - } - - if (use_gpu_fallback || !expert_weights.get_partial_shape().is_static() || !activations.get_partial_shape().is_static() || - !ids.get_partial_shape().is_static()) { - return rename_outputs_with_suffix({translate_mul_mat_id_gather_matmul_fallback(context, expert_weights, activations, ids)}, - context.get_name()); + const auto activations_type = ggml_openvino_get_device_name() == "GPU" ? ov::element::f16 : ov::element::f32; + if (activations.get_element_type() != activations_type) { + activations = std::make_shared<ov::op::v0::Convert>(activations, activations_type); } // GatherMatmul's A input is [n_used_or_1, n_tokens, k]; activations_3d is diff --git a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp index 41d7c54ae6be..9d4315aa4abc 100644 --- a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp @@ -29,19 +29,11 @@ OutputVector translate_mulmat(const NodeContext & context) { int op_case = context.get_op_case(); - ov::Output<Node> res; - ov::Output<ov::Node> B; - ov::Output<ov::Node> A; - if (op_case == 3) { - B = process_view_input(context, 0); - A = process_view_input(context, 1); - } else { - B = process_view_input_new(context, 0); - A = process_view_input_new(context, 1); - } + ov::Output<ov::Node> B = process_view_input_new(context, 0); + ov::Output<ov::Node> A = process_view_input_new(context, 1); if (A.get_element_type() != B.get_element_type()) { - B = std::make_shared<ov::op::v0::Convert>(context.get_input(0), context.get_input_type(1)); + B = std::make_shared<ov::op::v0::Convert>(B, context.get_input_type(1)); } auto B_shape = context.get_input_shape(0).to_shape(); @@ -84,7 +76,7 @@ OutputVector translate_mulmat(const NodeContext & context) { } bool transpose_b = true; - res = std::make_shared<ov::op::v0::MatMul>(A, B, false, transpose_b); + ov::Output<Node> res = std::make_shared<ov::op::v0::MatMul>(A, B, false, transpose_b); const auto output_type = context.get_output_type(); if (res.get_element_type() != output_type) { diff --git a/ggml/src/ggml-openvino/openvino/op/norm.cpp b/ggml/src/ggml-openvino/openvino/op/norm.cpp index c8bedb6dbf59..8660c6521b70 100644 --- a/ggml/src/ggml-openvino/openvino/op/norm.cpp +++ b/ggml/src/ggml-openvino/openvino/op/norm.cpp @@ -2,15 +2,10 @@ #include "../op_table.h" #include "../utils.h" +#include <cstring> #include <memory> -#include <openvino/op/add.hpp> #include <openvino/op/constant.hpp> -#include <openvino/op/divide.hpp> -#include <openvino/op/multiply.hpp> -#include <openvino/op/power.hpp> -#include <openvino/op/reduce_mean.hpp> -#include <openvino/op/sqrt.hpp> -#include <openvino/op/subtract.hpp> +#include <openvino/op/mvn.hpp> namespace ov { namespace frontend { @@ -21,33 +16,11 @@ OutputVector translate_norm(const NodeContext & context) { num_inputs_check(context, 1, 1); auto input_node = process_view_input_new(context, 0); - - // Step 1: Calculate mean along the last dimension - // mean = reduce_mean(input, axis=-1, keepdims=true) - auto mean = std::make_shared<ov::op::v1::ReduceMean>( - input_node, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); - - // Step 2: Calculate (input - mean) - auto centered = std::make_shared<ov::op::v1::Subtract>(input_node, mean); - - // Step 3: Calculate squared differences (input - mean)^2 - auto squared = std::make_shared<ov::op::v1::Power>( - centered, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {2.0f})); - - // Step 4: Calculate variance = mean((input - mean)^2) - auto variance = std::make_shared<ov::op::v1::ReduceMean>( - squared, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); - - // Step 5: Get epsilon from op_params float eps; memcpy(&eps, context.get_output_op_params(), sizeof(float)); - // Step 6: Calculate std = sqrt(variance + eps) - auto std_dev = std::make_shared<ov::op::v0::Sqrt>(std::make_shared<ov::op::v1::Add>( - variance, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps}))); - - // Step 7: Normalize: output = (input - mean) / std - auto res = std::make_shared<ov::op::v1::Divide>(centered, std_dev); + auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); + auto res = std::make_shared<ov::op::v6::MVN>(input_node, axes, true, eps, ov::op::MVNEpsMode::INSIDE_SQRT); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/pad.cpp b/ggml/src/ggml-openvino/openvino/op/pad.cpp index 492033d1b787..ae3d7be18eca 100644 --- a/ggml/src/ggml-openvino/openvino/op/pad.cpp +++ b/ggml/src/ggml-openvino/openvino/op/pad.cpp @@ -8,6 +8,7 @@ #include <openvino/op/pad.hpp> #include <openvino/op/reshape.hpp> #include <openvino/op/shape_of.hpp> +#include <utility> #include <vector> namespace ov { @@ -20,7 +21,7 @@ namespace { ov::Output<ov::Node> translate_circular_pad(ov::Output<ov::Node> input, const std::array<int32_t, 8> & pads, const ov::Shape & input_shape) { - ov::Output<ov::Node> result = input; + ov::Output<ov::Node> result = std::move(input); const std::array<int32_t, 4> pads_begin = {pads[6], pads[4], pads[2], pads[0]}; const std::array<int32_t, 4> pads_end = {pads[7], pads[5], pads[3], pads[1]}; @@ -60,9 +61,7 @@ OutputVector translate_pad(const NodeContext & context) { auto input = process_view_input_new(context, 0); if (context.get_input_shape(0) == context.get_output_shape()) { - auto input_shape = std::make_shared<ov::op::v3::ShapeOf>(input); - auto res = std::make_shared<ov::op::v1::Reshape>(input, input_shape, false); - return rename_outputs_with_suffix({res}, context.get_name()); + return rename_outputs_with_suffix({input}, context.get_name()); } const int32_t * op_params = context.get_output_op_params(); diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp index 85550bff396b..df4f038984c5 100644 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp @@ -45,11 +45,22 @@ OutputVector translate_permute(const NodeContext & context) { static_cast<int64_t>(perm_values.size() - 1 - input_axis); } } - auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); - if (op_case == 1 || context.is_stateful()) { + // The stateful path carries hidden-state tensors in a rank-3 layout (the + // leading batch dim is dropped, e.g. Gemma4's per-layer-embedding path). The + // perm above is rank-4; when the actual input is rank-3, drop the batch axis + // (perm[0], which is always the identity 0 here) and shift the rest down by 1 + // so the transpose order matches the input rank. + std::vector<int64_t> perm_used = perm_values; + const auto & src_ps = src.get_partial_shape(); + if (src_ps.rank().is_static() && src_ps.rank().get_length() == 3 && perm_values.size() == 4 && + perm_values[0] == 0) { + perm_used = {perm_values[1] - 1, perm_values[2] - 1, perm_values[3] - 1}; + } + auto perm = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{perm_used.size()}, perm_used); res = std::make_shared<ov::op::v1::Transpose>(src, perm); } else if (op_case == 2) { + auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); auto output_shape = context.get_output_shape().to_shape(); auto n_heads = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[1]}); auto head_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); @@ -68,6 +79,7 @@ OutputVector translate_permute(const NodeContext & context) { auto reshaped = std::make_shared<ov::op::v1::Reshape>(src, new_shape, true); res = std::make_shared<ov::op::v1::Transpose>(reshaped, perm); } else { + auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); auto cache_shape = src.get_partial_shape(); auto output_shape = context.get_output_shape().to_shape(); int64_t head_size = output_shape[3]; diff --git a/ggml/src/ggml-openvino/openvino/op/repeat.cpp b/ggml/src/ggml-openvino/openvino/op/repeat.cpp index d58b59e4e309..b7aeaa24fa82 100644 --- a/ggml/src/ggml-openvino/openvino/op/repeat.cpp +++ b/ggml/src/ggml-openvino/openvino/op/repeat.cpp @@ -1,7 +1,6 @@ #include "../node_context.h" #include "../op_table.h" #include "../utils.h" -#include "ggml.h" #include <memory> #include <openvino/op/broadcast.hpp> diff --git a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp index 9cbce7db0d50..25c9535454be 100644 --- a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +++ b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp @@ -25,9 +25,7 @@ OutputVector translate_rms_norm(const NodeContext & context) { auto op_case = context.get_op_case(); ov::Output<ov::Node> input_node; - if (op_case == 1) { - input_node = process_view_input_new(context, 0); - } else if (op_case == 2) { + if (op_case == 2) { auto ssm_state_size = context.get_ssm_state_size(); // The GDN op packs [attn | new_state] along the row axis; the state occupies the last // ssm_state_size * n_seqs rows. Slice it off (scaling by the active sequence count) to keep diff --git a/ggml/src/ggml-openvino/openvino/op/rope.cpp b/ggml/src/ggml-openvino/openvino/op/rope.cpp index 8f20a0d196eb..a3da7d1fbb4f 100644 --- a/ggml/src/ggml-openvino/openvino/op/rope.cpp +++ b/ggml/src/ggml-openvino/openvino/op/rope.cpp @@ -11,16 +11,11 @@ #include <openvino/op/concat.hpp> #include <openvino/op/constant.hpp> #include <openvino/op/convert.hpp> -#include <openvino/op/cos.hpp> -#include <openvino/op/gather.hpp> #include <openvino/op/multiply.hpp> #include <openvino/op/reshape.hpp> -#include <openvino/op/shape_of.hpp> -#include <openvino/op/sin.hpp> #include <openvino/op/slice.hpp> #include <openvino/op/split.hpp> #include <openvino/op/subtract.hpp> -#include <openvino/op/transpose.hpp> #include <openvino/op/unsqueeze.hpp> #include <openvino/op/variadic_split.hpp> #include <vector> @@ -37,13 +32,14 @@ OutputVector translate_rope(const NodeContext & context) { ov::Output<Node> res; - auto data_node = context.get_input(0).get_node_shared_ptr(); + auto data_node = process_view_input_new(context, 0).get_node_shared_ptr(); auto output_shape = context.get_output_shape().to_shape(); int32_t * op_params = context.get_output_op_params(); const int mode = op_case; const int64_t head_dim = static_cast<int64_t>(output_shape[3]); const int64_t configured_n_dims = static_cast<int64_t>(op_params[1]); const int64_t n_dims = configured_n_dims == 0 ? head_dim : configured_n_dims; + const int64_t n_offs = static_cast<int64_t>(op_params[15]); constexpr int TYPE_NORMAL = 0; constexpr int TYPE_NEOX = 1; @@ -55,27 +51,27 @@ OutputVector translate_rope(const NodeContext & context) { cos_theta_node = context.get_input("rope_cos"); sin_theta_node = context.get_input("rope_sin"); } else { - auto inp_pos = context.get_input(1).get_node_shared_ptr(); - std::shared_ptr<ov::Node> rope_freqs_weight; + std::string cache_key = "rope_sin_cos"; + for (int i = 0; i < 15; i++) { + cache_key += "_" + std::to_string(op_params[i]); + } if (context.get_input_size() == 3) { - rope_freqs_weight = context.get_input(2).get_node_shared_ptr(); + cache_key += "_ff_" + context.get_input_names()[2]; } - auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode == TYPE_IMROPE, false); - sin_theta_node = sin_cos.first; - cos_theta_node = sin_cos.second; - } - - if (context.get_view_input_size(0) > 0) { - data_node = process_view_input_new(context, 0).get_node_shared_ptr(); - if (context.is_stateful()) { - auto data_shape = ov::op::v0::Constant::create( - ov::element::i64, {3}, std::vector<int64_t>{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - data_node = std::make_shared<ov::op::v1::Reshape>(data_node, data_shape, false); + if (context.has_input(cache_key + "_cos")) { + cos_theta_node = context.get_input(cache_key + "_cos"); + sin_theta_node = context.get_input(cache_key + "_sin"); } else { - auto data_shape = ov::op::v0::Constant::create( - ov::element::i64, {4}, - std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - data_node = std::make_shared<ov::op::v1::Reshape>(data_node, data_shape, false); + auto inp_pos = context.get_input(1).get_node_shared_ptr(); + std::shared_ptr<ov::Node> rope_freqs_weight; + if (context.get_input_size() == 3) { + rope_freqs_weight = context.get_input(2).get_node_shared_ptr(); + } + auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode == TYPE_IMROPE, false); + sin_theta_node = sin_cos.first; + cos_theta_node = sin_cos.second; + context.put_shared(cache_key + "_cos", cos_theta_node); + context.put_shared(cache_key + "_sin", sin_theta_node); } } @@ -84,52 +80,34 @@ OutputVector translate_rope(const NodeContext & context) { data_node = std::make_shared<ov::op::v0::Convert>(data_node, ov::element::f32); } - FRONT_END_OP_CONVERSION_CHECK(n_dims > 0 && n_dims <= head_dim && (n_dims % 2 == 0), - "ROPE expects even n_dims in [1, head_dim]"); - - // TODO(openvino-gpu-rope-fusion): TEMPORARY WORKAROUND - do NOT revert until the - // OpenVINO GPU plugin is updated. - // + FRONT_END_OP_CONVERSION_CHECK(n_offs >= 0 && (n_offs % 2 == 0), + "ROPE expects non-negative even n_offs"); + FRONT_END_OP_CONVERSION_CHECK(n_dims > 0 && n_dims + n_offs <= head_dim && (n_dims % 2 == 0), + "ROPE expects even n_dims in [1, head_dim - n_offs]"); + + // RoPEFusionFlux requires rank_equals(4) on x, t_cos and t_sin. The cos/sin + // tables are already built rank-4 ([1, S, 1, head_size/2]) for both modes. In + // stateful mode the data arrives rank-3 ([S, n_heads, head_size]), so lift it + // to rank-4 ([1, S, n_heads, head_size]) here. Stateful RoPE already produced + // rank-4 output, so downstream attention is unaffected. + if (context.is_stateful()) { + auto r4_shape = ov::op::v0::Constant::create( + ov::element::i64, {4}, + std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); + data_node = std::make_shared<ov::op::v1::Reshape>(data_node, r4_shape, false); + } // For TYPE_NORMAL rope (both stateful and stateless) we emit the Flux-style // interleaved pattern below so the GPU plugin's RoPEFusionFlux matcher folds it - // into ov::op::internal::RoPE. The matcher requires rank-4 inputs, which is why - // the original even/odd Slice translation (kept in the `else if (mode == - // TYPE_NORMAL)` branch below for reference) does not get fused. - // - // Once the GPU plugin's RoPE fusion is extended to also recognize the original - // even/odd Slice form, this Flux rewrite should be removed and both modes should - // be restored to the captured even/odd translation. Until then, keep both paths: - // the active Flux rewrite here and the previous translation preserved below. + // into ov::op::internal::RoPE. if (mode == TYPE_NORMAL) { auto axis_last = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto step_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - // Emit the Flux-style interleaved-RoPE pattern so the GPU plugin's - // RoPEFusionFlux matcher folds this subgraph into ov::op::internal::RoPE: - // x_paired = Reshape(x_rot, [1, S, n_heads, n_dims/2, 2]) - // x0, x1 = Split(x_paired, axis=-1, num_splits=2) - // x1_neg = x1 * -1 - // x_rotated = Reshape(Concat([x1_neg, x0], axis=-1), [1, S, n_heads, n_dims]) - // y_rot = x_rot * t_cos + x_rotated * t_sin - // y = Concat([y_rot, x_tail], axis=-1) if n_dims < head_dim - // Mathematically equivalent to the even/odd Slice form below. - // - // RoPEFusionFlux requires rank_equals(4) on x, t_cos and t_sin. The cos/sin - // tables are already built rank-4 ([1, S, 1, head_size/2]) for both modes. In - // stateful mode the data arrives rank-3 ([S, n_heads, head_size]), so lift it - // to rank-4 ([1, S, n_heads, head_size]) here. Stateful RoPE already produced - // rank-4 output, so downstream attention is unaffected. - if (context.is_stateful()) { - auto r4_shape = ov::op::v0::Constant::create( - ov::element::i64, {4}, - std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - data_node = std::make_shared<ov::op::v1::Reshape>(data_node, r4_shape, false); - } const int64_t n_heads = static_cast<int64_t>(output_shape[2]); const int64_t half = n_dims / 2; - auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_dims}); - auto rot_data = std::make_shared<ov::op::v8::Slice>(data_node, zero, rot_end, step_one, axis_last); + auto rot_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); + auto rot_data = std::make_shared<ov::op::v8::Slice>(data_node, rot_start, rot_end, step_one, axis_last); auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f}); @@ -153,7 +131,7 @@ OutputVector translate_rope(const NodeContext & context) { // Expand cos/sin from [..., n_dims/2] to [..., n_dims] by repeating each // entry twice. Use special_zero on the final Reshape so the seq dim passes // through dynamically. Final rank is 4 to satisfy the matcher's predicate. - auto expand_cos_sin = [&](Output<Node> cs) { + auto expand_cos_sin = [&](const Output<Node>& cs) { auto cs_unsq = std::make_shared<ov::op::v0::Unsqueeze>( cs, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1})); auto bcast_target = ov::op::v0::Constant::create( @@ -170,123 +148,80 @@ OutputVector translate_rope(const NodeContext & context) { auto y2 = std::make_shared<ov::op::v1::Multiply>(x_rotated, sin_full); auto rotated = std::make_shared<ov::op::v1::Add>(y1, y2); - if (n_dims < head_dim) { - auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_dims}); + ov::OutputVector concat_parts; + if (n_offs > 0) { + auto head_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto head_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto head = std::make_shared<ov::op::v8::Slice>(data_node, head_start, head_end, step_one, axis_last); + concat_parts.push_back(head); + } + concat_parts.push_back(rotated); + if (n_offs + n_dims < head_dim) { + auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); auto tail_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim}); auto tail = std::make_shared<ov::op::v8::Slice>(data_node, tail_start, tail_end, step_one, axis_last); - res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{rotated, tail}, -1); - } else { - res = rotated; + concat_parts.push_back(tail); } - } - // PRESERVED PREVIOUS TRANSLATION - Re-enable this branch (and remove the Flux branch above) once - // the GPU plugin's RoPE fusion is updated to recognize the even/odd Slice form; - // see the TODO(openvino-gpu-rope-fusion) note above. Do not delete. - // - // Original even/odd Slice form. In stateless mode it ran on rank-4 data - // ([1, S, n_heads, head_size]); in stateful mode on rank-3 data - // ([S, n_heads, head_size]). Either way it does not match RoPEFusionFlux - // (which needs rank-4 x in the interleaved layout), so the RoPE stays as - // discrete elementwise ops. - // - // } else if (mode == TYPE_NORMAL) { - // auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - // auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - // auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - // auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); - // auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); - // Output<Node> even_slice; - // Output<Node> odd_slice; - // // stateful data is rank 3 (unsqueeze at axis 3), stateless is rank 4 (axis 4) - // int32_t unsqueeze_dim = context.is_stateful() ? 3 : 4; - // even_slice = std::make_shared<ov::op::v8::Slice>(data_node, zero, end, two, neg_one); - // odd_slice = std::make_shared<ov::op::v8::Slice>(data_node, one, end, two, neg_one); - // - // Output<Node> first_half = - // std::make_shared<ov::op::v1::Subtract>(std::make_shared<ov::op::v1::Multiply>(even_slice, cos_theta_node), - // std::make_shared<ov::op::v1::Multiply>(odd_slice, sin_theta_node)); - // Output<Node> second_half = - // std::make_shared<ov::op::v1::Add>(std::make_shared<ov::op::v1::Multiply>(even_slice, sin_theta_node), - // std::make_shared<ov::op::v1::Multiply>(odd_slice, cos_theta_node)); - // - // first_half = std::make_shared<ov::op::v0::Unsqueeze>(first_half, - // ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim})); - // second_half = std::make_shared<ov::op::v0::Unsqueeze>(second_half, - // ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim})); - // auto stack = std::make_shared<ov::op::v0::Concat>(OutputVector{first_half, second_half}, unsqueeze_dim); - // - // auto data_shape = ov::op::v0::Constant::create( - // ov::element::i64, {4}, std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - // res = std::make_shared<ov::op::v1::Reshape>(stack, data_shape, false); - else if (mode == TYPE_NEOX) { - // In stateful mode the data arrives rank-3 ([S, n_heads, head_size]) while the - // cos/sin tables are rank-4 ([1, S, 1, n_dims/2]). The resulting mixed-rank - // broadcast in the Multiply below is miscomputed by the OpenVINO GPU plugin, - // corrupting the rotated Q/K. Lift the data to rank-4 ([1, S, n_heads, head_size]) - // first so the RoPE Multiplies are equal-rank, matching the TYPE_NORMAL branch. - // Stateful RoPE already produced rank-4 output, so downstream attention is unaffected. - if (context.is_stateful()) { - auto r4_shape = ov::op::v0::Constant::create( - ov::element::i64, {4}, - std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - data_node = std::make_shared<ov::op::v1::Reshape>(data_node, r4_shape, false); + if (concat_parts.size() == 1) { + res = rotated; + } else { + res = std::make_shared<ov::op::v0::Concat>(concat_parts, -1); } - auto axis_last = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1}); - std::vector<int64_t> split_lengths = {n_dims / 2, n_dims / 2}; - if (n_dims < head_dim) { - split_lengths.push_back(head_dim - n_dims); + } else if (mode == TYPE_NEOX || mode == TYPE_IMROPE) { + if (mode == TYPE_IMROPE) { + auto cos_sin_shape = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{4}, + std::vector<int64_t>{1, -1, 1, (n_dims >> 1)}); + cos_theta_node = std::make_shared<ov::op::v1::Reshape>(cos_theta_node, cos_sin_shape, true); + sin_theta_node = std::make_shared<ov::op::v1::Reshape>(sin_theta_node, cos_sin_shape, true); } - auto data_split = std::make_shared<ov::op::v1::VariadicSplit>( - data_node, axis_last, - ov::op::v0::Constant::create(ov::element::i64, {split_lengths.size()}, split_lengths)); - Output<Node> slice_data_node_0 = data_split->outputs()[0]; - Output<Node> slice_data_node_1 = data_split->outputs()[1]; - - auto first_half_node = std::make_shared<ov::op::v1::Subtract>( - std::make_shared<ov::op::v1::Multiply>(slice_data_node_0, cos_theta_node), - std::make_shared<ov::op::v1::Multiply>(slice_data_node_1, sin_theta_node)); - - auto second_half_node = std::make_shared<ov::op::v1::Add>( - std::make_shared<ov::op::v1::Multiply>(slice_data_node_0, sin_theta_node), - std::make_shared<ov::op::v1::Multiply>(slice_data_node_1, cos_theta_node)); + auto axis_last = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + auto step_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - if (n_dims < head_dim) { - Output<Node> tail = data_split->outputs()[2]; - res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{first_half_node, second_half_node, tail}, -1); - } else { - res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{first_half_node, second_half_node}, -1); + Output<Node> rot_data = data_node; + if (n_offs > 0 || n_offs + n_dims < head_dim) { + auto rot_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); + rot_data = std::make_shared<ov::op::v8::Slice>(data_node, rot_start, rot_end, step_one, axis_last); } - } else if (mode == TYPE_IMROPE) { - auto cos_sin_shape = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{4}, - std::vector<int64_t>{1, -1, 1, (n_dims >> 1)}); - auto cos_reshaped = std::make_shared<ov::op::v1::Reshape>(cos_theta_node, cos_sin_shape, true); - auto sin_reshaped = std::make_shared<ov::op::v1::Reshape>(sin_theta_node, cos_sin_shape, true); + + const int64_t half = n_dims / 2; + auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f}); auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {3}); - std::vector<int64_t> split_lengths = {n_dims / 2, n_dims / 2}; - if (n_dims < head_dim) { - split_lengths.push_back(head_dim - n_dims); - } + auto split_lengths = ov::op::v0::Constant::create(ov::element::i64, {2}, {half, half}); + auto data_split = std::make_shared<ov::op::v1::VariadicSplit>(rot_data, split_axis, split_lengths); + Output<Node> x1 = data_split->outputs()[0]; + Output<Node> x2 = data_split->outputs()[1]; + + auto x2_neg = std::make_shared<ov::op::v1::Multiply>(x2, neg_one_f); + auto x_rotate_half = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{x2_neg, x1}, -1); - auto split_a = std::make_shared<ov::op::v1::VariadicSplit>( - data_node, split_axis, - ov::op::v0::Constant::create(ov::element::i64, {split_lengths.size()}, split_lengths)); - auto x0 = split_a->output(0); - auto x1 = split_a->output(1); - auto mul_a = std::make_shared<ov::op::v1::Multiply>(x0, cos_reshaped); - auto mul_b = std::make_shared<ov::op::v1::Multiply>(x1, sin_reshaped); - auto sub = std::make_shared<ov::op::v1::Subtract>(mul_a, mul_b); + auto cos_full = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{cos_theta_node, cos_theta_node}, -1); + auto sin_full = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{sin_theta_node, sin_theta_node}, -1); - auto mul_c = std::make_shared<ov::op::v1::Multiply>(x0, sin_reshaped); - auto mul_d = std::make_shared<ov::op::v1::Multiply>(x1, cos_reshaped); - auto add = std::make_shared<ov::op::v1::Add>(mul_c, mul_d); + auto y1 = std::make_shared<ov::op::v1::Multiply>(rot_data, cos_full); + auto y2 = std::make_shared<ov::op::v1::Multiply>(x_rotate_half, sin_full); + auto rotated = std::make_shared<ov::op::v1::Add>(y1, y2); - if (n_dims < head_dim) { - auto tail = split_a->output(2); - res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{sub, add, tail}, 3); + ov::OutputVector concat_parts; + if (n_offs > 0) { + auto head_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto head_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto head = std::make_shared<ov::op::v8::Slice>(data_node, head_start, head_end, step_one, axis_last); + concat_parts.push_back(head); + } + concat_parts.push_back(rotated); + if (n_offs + n_dims < head_dim) { + auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); + auto tail_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim}); + auto tail = std::make_shared<ov::op::v8::Slice>(data_node, tail_start, tail_end, step_one, axis_last); + concat_parts.push_back(tail); + } + if (concat_parts.size() == 1) { + res = rotated; } else { - res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{sub, add}, 3); + res = std::make_shared<ov::op::v0::Concat>(concat_parts, -1); } } diff --git a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp index 0fe8e0a8d067..3b606c82accb 100644 --- a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp @@ -7,6 +7,7 @@ #include <memory> #include <openvino/core/node.hpp> #include <openvino/core/node_output.hpp> +#include <openvino/core/type.hpp> #include <openvino/frontend/exception.hpp> #include <openvino/op/broadcast.hpp> #include <openvino/op/concat.hpp> @@ -75,7 +76,7 @@ OutputVector translate_set_rows(const NodeContext & context) { res = std::make_shared<ov::op::v3::ScatterUpdate>(dst, ind_squeezed, data_reshaped, axes); } - auto dst_reshape = std::dynamic_pointer_cast<ov::op::v1::Reshape>(dst.get_node_shared_ptr()); + auto dst_reshape = ov::as_type_ptr<ov::op::v1::Reshape>(dst.get_node_shared_ptr()); if (!multidim_indices && dst_reshape) { // Fix the case of multiple sequences, reshape back to original shape [1, n_seq, ctx_per_seq, emb] // ctx_per_seq is not fixed due to llama-bench compatibility diff --git a/ggml/src/ggml-openvino/openvino/op/transpose.cpp b/ggml/src/ggml-openvino/openvino/op/transpose.cpp index 8d89ca556d68..0651a410a1a0 100644 --- a/ggml/src/ggml-openvino/openvino/op/transpose.cpp +++ b/ggml/src/ggml-openvino/openvino/op/transpose.cpp @@ -14,9 +14,7 @@ OutputVector translate_transpose(const NodeContext & context) { // Compute permute order from input/output shape and stride information // so it adapts to different input and output layouts. - auto input_shape = context.get_input_shape(0).to_shape(); auto input_stride = context.get_input_stride(0); - auto output_shape = context.get_output_shape().to_shape(); auto output_stride = context.get_output_stride(); // Compute permute order by matching output and input stride rankings. diff --git a/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp b/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp deleted file mode 100644 index 48ee0431ff76..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include <openvino/core/node_output.hpp> -#include <openvino/op/multiply.hpp> -#include <openvino/op/sigmoid.hpp> - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_unary_silu(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = process_view_input_new(context, 0); - auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(input); - auto res = std::make_shared<ov::op::v1::Multiply>(input, sigmoid); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp b/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp index 756d9c33d736..a9e495c372f7 100644 --- a/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +++ b/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp @@ -1,6 +1,7 @@ #include "../node_context.h" #include "../op_table.h" #include "../utils.h" +#include "ggml-openvino/ggml-openvino-extra.h" #include <openvino/op/abs.hpp> #include <openvino/op/add.hpp> @@ -9,6 +10,7 @@ #include <openvino/op/log.hpp> #include <openvino/op/negative.hpp> #include <openvino/op/relu.hpp> +#include <openvino/op/softplus.hpp> namespace ov { namespace frontend { @@ -18,6 +20,10 @@ namespace op { OutputVector translate_unary_softplus(const NodeContext & context) { num_inputs_check(context, 1, 1); + if (ggml_openvino_getenv_int("GGML_OPENVINO_NATIVE_SOFTPLUS") != 0) { + return translate_1to1_match_1_input<ov::op::v4::SoftPlus>(context); + } + auto input = process_view_input_new(context, 0); const auto element_type = input.get_element_type(); auto one = ov::op::v0::Constant::create(element_type, ov::Shape{}, {1.0f}); diff --git a/ggml/src/ggml-openvino/openvino/op/view.cpp b/ggml/src/ggml-openvino/openvino/op/view.cpp index 56f5ceec9bb0..ca2d2dc08732 100644 --- a/ggml/src/ggml-openvino/openvino/op/view.cpp +++ b/ggml/src/ggml-openvino/openvino/op/view.cpp @@ -7,7 +7,6 @@ #include <openvino/op/reshape.hpp> #include <openvino/op/shape_of.hpp> #include <openvino/op/slice.hpp> -#include <set> namespace ov { namespace frontend { @@ -153,7 +152,8 @@ OutputVector translate_view(const NodeContext & context) { return {input}; } - int64_t src_elems = 1, dst_elems = 1; + int64_t src_elems = 1; + int64_t dst_elems = 1; for (int64_t i = 0; i < src_shape.rank().get_length(); ++i) { if (src_shape[i].is_dynamic()) { return {input}; diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index d4f5ac307329..f249a06bb8a0 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -13,6 +13,7 @@ #include <openvino/op/relu.hpp> #include <openvino/op/sigmoid.hpp> #include <openvino/op/subtract.hpp> +#include <openvino/op/swish.hpp> #include <openvino/op/tanh.hpp> namespace ov { @@ -50,10 +51,9 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() { {"GGML_OP_TRANSPOSE", op::translate_transpose }, {"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input<v7::Gelu> }, {"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input<v0::Sigmoid> }, - {"GGML_UNARY_OP_SILU", op::translate_unary_silu }, + {"GGML_UNARY_OP_SILU", op::translate_1to1_match_1_input<v4::Swish> }, {"GGML_UNARY_OP_SOFTPLUS", op::translate_unary_softplus }, {"GGML_UNARY_OP_TANH", op::translate_1to1_match_1_input<v0::Tanh> }, - {"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input<v0::Sigmoid> }, {"GGML_UNARY_OP_EXP", op::translate_1to1_match_1_input<v0::Exp> }, {"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input<v0::Negative> }, {"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input<v0::Relu> }, diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h index a0a42bff337d..3dc98bd96763 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ b/ggml/src/ggml-openvino/openvino/op_table.h @@ -30,7 +30,6 @@ GGML_OP_CONVERTER(translate_sqr); GGML_OP_CONVERTER(translate_rope); GGML_OP_CONVERTER(translate_scale); GGML_OP_CONVERTER(translate_sqrt); -GGML_OP_CONVERTER(translate_unary_silu); GGML_OP_CONVERTER(translate_unary_softplus); GGML_OP_CONVERTER(translate_soft_max); GGML_OP_CONVERTER(translate_transpose); diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp new file mode 100644 index 000000000000..c4872ac2e9c5 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp @@ -0,0 +1,273 @@ +#include "fuse_moe_compressed.h" + +#include <limits> +#include <set> +#include <memory> +#include <openvino/core/graph_util.hpp> +#include <openvino/core/rt_info.hpp> +#include <openvino/op/constant.hpp> +#include <openvino/op/convert.hpp> +#include <openvino/op/multiply.hpp> +#include <openvino/op/reduce_sum.hpp> +#include <openvino/op/reshape.hpp> +#include <openvino/op/squeeze.hpp> +#include <openvino/op/subtract.hpp> +#include <openvino/op/swish.hpp> +#include <openvino/op/transpose.hpp> +#include <openvino/op/unsqueeze.hpp> +#include <openvino/pass/constant_folding.hpp> +#include <openvino/pass/pattern/op/optional.hpp> +#include <openvino/pass/pattern/op/wrap_type.hpp> + +#include "../op/gather_matmul.hpp" +#include "../op/moe_compressed.hpp" + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +namespace { + +struct dequant_inputs { + ov::Output<ov::Node> weight; + ov::Output<ov::Node> scale; + ov::Output<ov::Node> zp; + bool has_zp = false; + bool ok = false; +}; + +// Peel the chain built by make_int4_weights/make_int8_weights back to its Constant inputs. +// Grouped weights keep the pre-Reshape rank-4 form [n_expert, n, k/group, group] with scale +// and zp at [n_expert, n, k/group, 1], which is the layout MOECompressed expects. Channel-wise +// weights stay rank-3 with a rank-3 scale and carry no zp. +dequant_inputs unwrap_dequant(const ov::Output<ov::Node> & b) { + dequant_inputs res; + + auto node = b.get_node_shared_ptr(); + while (ov::is_type<ov::op::v0::Convert>(node) || ov::is_type<ov::op::v1::Reshape>(node)) { + node = node->get_input_node_shared_ptr(0); + } + + auto mul = ov::as_type_ptr<ov::op::v1::Multiply>(node); + if (!mul) { + return res; + } + res.scale = mul->input_value(1); + + auto lhs = mul->get_input_node_shared_ptr(0); + if (auto sub = ov::as_type_ptr<ov::op::v1::Subtract>(lhs)) { + // Take the zero point down to its Constant: an integer zp is wrapped in a Convert to f16, + // and the op wants the integer form. A natively quantized expert instead carries an exact + // f16 zp (-min/scale) with no integer behind it, which the MoE kernel does not accept. + auto zp_node = sub->get_input_node_shared_ptr(1); + while (ov::is_type<ov::op::v0::Convert>(zp_node)) { + zp_node = zp_node->get_input_node_shared_ptr(0); + } + res.zp = zp_node->output(0); + res.has_zp = true; + lhs = sub->get_input_node_shared_ptr(0); + } + while (ov::is_type<ov::op::v0::Convert>(lhs)) { + lhs = lhs->get_input_node_shared_ptr(0); + } + if (!ov::is_type<ov::op::v0::Constant>(lhs)) { + return res; + } + + res.weight = lhs->output(0); + res.ok = res.scale.get_partial_shape().is_static() && res.weight.get_partial_shape().is_static(); + return res; +} + +size_t logical_k(const ov::Shape & shape) { + return shape.size() == 4 ? shape[2] * shape[3] : shape.back(); +} + +} // namespace + +FuseMoeCompressed::FuseMoeCompressed() { + using namespace ov::pass::pattern; + + // The gate and up projections each get their own Reshape/Transpose of the hidden state and + // their own Reshape of the routing ids, so every branch needs its own sub-pattern. On GPU + // mul_mat_id also converts the activations to f16 before the op and back to f32 after it, + // so those Converts are matched as optional. + auto hidden_gate_m = any_input(); + auto a_gate_reshape_m = wrap_type<ov::op::v1::Reshape>({ hidden_gate_m, any_input() }); + auto a_gate_m = + wrap_type<ov::op::v1::Transpose>({ optional<ov::op::v0::Convert>({ a_gate_reshape_m }), any_input() }); + auto hidden_up_m = any_input(); + auto a_up_m = wrap_type<ov::op::v1::Transpose>( + { optional<ov::op::v0::Convert>({ wrap_type<ov::op::v1::Reshape>({ hidden_up_m, any_input() }) }), + any_input() }); + + auto gate_w_m = any_input(); + auto up_w_m = any_input(); + auto down_w_m = any_input(); + auto ids_gate_m = any_input(); + auto ids_up_m = any_input(); + auto ids_down_m = any_input(); + + auto bgm_gate_m = wrap_type<ov::op::internal::GatherMatmul>({ a_gate_m, gate_w_m, ids_gate_m, any_input() }); + auto gate_u_m = optional<ov::op::v0::Convert>({ wrap_type<ov::op::v0::Unsqueeze>( + { wrap_type<ov::op::v1::Transpose>({ bgm_gate_m, any_input() }), any_input() }) }); + + auto silu_m = wrap_type<ov::op::v4::Swish>({ gate_u_m }); + + auto bgm_up_m = wrap_type<ov::op::internal::GatherMatmul>({ a_up_m, up_w_m, ids_up_m, any_input() }); + auto up_u_m = optional<ov::op::v0::Convert>({ wrap_type<ov::op::v0::Unsqueeze>( + { wrap_type<ov::op::v1::Transpose>({ bgm_up_m, any_input() }), any_input() }) }); + auto swiglu_m = wrap_type<ov::op::v1::Multiply>({ silu_m, up_u_m }); + + auto d_t_m = wrap_type<ov::op::v1::Transpose>( + { optional<ov::op::v0::Convert>({ wrap_type<ov::op::v1::Reshape>({ swiglu_m, any_input() }) }), + any_input() }); + auto bgm_down_m = wrap_type<ov::op::internal::GatherMatmul>({ d_t_m, down_w_m, ids_down_m, any_input() }); + auto down_u_m = optional<ov::op::v0::Convert>({ wrap_type<ov::op::v0::Unsqueeze>( + { wrap_type<ov::op::v1::Transpose>({ bgm_down_m, any_input() }), any_input() }) }); + + auto routing_m = any_input(); + auto weighted_m = wrap_type<ov::op::v1::Multiply>({ down_u_m, routing_m }); + auto root_m = wrap_type<ov::op::v1::ReduceSum>({ weighted_m, any_input() }); + + const auto callback = [=](Matcher & m) { + auto & pm = m.get_pattern_value_map(); + + const auto gate = unwrap_dequant(pm.at(gate_w_m)); + const auto up = unwrap_dequant(pm.at(up_w_m)); + const auto down = unwrap_dequant(pm.at(down_w_m)); + if (!gate.ok || !up.ok || !down.ok) { + return false; + } + + const auto gate_shape = gate.weight.get_shape(); + const auto up_shape = up.weight.get_shape(); + const auto down_shape = down.weight.get_shape(); + if (gate_shape != up_shape || gate_shape.size() < 3 || down_shape.size() < 3) { + return false; + } + + // MOECompressed carries one group_size and one has_zp for all three projections, so a + // model whose down-proj is quantized differently from gate/up cannot be described. This + // happens when ggml requantizes Q5_K/Q6_K experts to channel-wise int8. + if (gate.has_zp != down.has_zp || gate_shape.size() != down_shape.size()) { + return false; + } + + // The kernel only takes an integer zero point (moe_3gemm_swiglu_opt validate_impl). + if (gate.has_zp) { + static const std::set<ov::element::Type> int_zp_types = { ov::element::u4, ov::element::i4, + ov::element::u8, ov::element::i8 }; + if (int_zp_types.count(gate.zp.get_element_type()) == 0 || + int_zp_types.count(down.zp.get_element_type()) == 0) { + return false; + } + } + + // Config holds a single group_size for all three projections. + const auto group_of = [](const dequant_inputs & w) { + const auto s = w.weight.get_shape(); + return s.size() == 4 ? s[3] : logical_k(s); + }; + if (group_of(gate) != group_of(up) || group_of(gate) != group_of(down)) { + return false; + } + + // all three branches must route the same hidden state through the same experts + if (pm.at(hidden_gate_m) != pm.at(hidden_up_m)) { + return false; + } + + auto ids = pm.at(ids_down_m); + const auto ids_pshape = ids.get_partial_shape(); + if (ids_pshape.rank().is_dynamic() || ids_pshape[ids_pshape.rank().get_length() - 1].is_dynamic()) { + return false; + } + const size_t top_k = ids_pshape[ids_pshape.rank().get_length() - 1].get_length(); + + // routing weights arrive as [1, n_tokens, top_k, 1]; the op wants [..., top_k] + auto routing = pm.at(routing_m); + const auto routing_pshape = routing.get_partial_shape(); + if (routing_pshape.rank().is_dynamic() || routing_pshape.rank().get_length() != 4 || + routing_pshape[3] != 1) { + return false; + } + // MOE requires routing weights and ids to have the same shape. Drop the trailing 1 of the + // routing weights and give the ids the leading batch dim, so both become [1, n_tokens, top_k]. + routing = std::make_shared<ov::op::v0::Squeeze>( + routing, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 3 })); + if (ids_pshape.rank().get_length() == 2) { + ids = std::make_shared<ov::op::v0::Unsqueeze>( + ids, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 0 })); + } + if (routing.get_partial_shape() != ids.get_partial_shape()) { + return false; + } + + const size_t down_k = logical_k(down_shape); + const auto down_scale_shape = down.scale.get_shape(); + const size_t down_groups = down_scale_shape.size() >= 3 ? down_scale_shape[2] : 1; + + ov::op::internal::MOECompressed::Config config; + config.expert_type = ov::op::internal::MOE::Expert_type::GEMM3_SWIGLU; + config.activation_type = ov::op::internal::MOE::Activation_type::SWIGLU; + config.expert_alpha = 0.0f; + config.expert_beta = 1.0f; + config.gate_idx = 0; + config.hidden_size = logical_k(gate_shape); + config.inter_size = gate_shape[1]; + config.num_expert = gate_shape[0]; + config.num_shared_expert = 0; + config.top_k = top_k; + config.group_size = down_groups <= 1 ? std::numeric_limits<size_t>::max() : down_k / down_groups; + config.has_batch_dim = true; + config.has_zp = gate.has_zp; + // dynamic makes the output follow the hidden state, so the plugin can lower this region + // to f16 together with the rest of the graph + config.out_type = ov::element::dynamic; + + auto absent_zp = [] { + auto zp = std::make_shared<ov::op::v0::Constant>(ov::element::dynamic, ov::Shape{ 0 }); + ov::pass::disable_constant_folding(zp); + return zp->output(0); + }; + + // MOE takes its output type from the hidden state. Transpose the activations before the + // f16 Convert that mul_mat_id adds on GPU, so the op stays f32 like the block it replaces + // and the plugin can lower the whole region uniformly. + const auto a_transpose = pm.at(a_gate_m).get_node_shared_ptr(); + ov::Output<ov::Node> hidden = + std::make_shared<ov::op::v1::Transpose>(pm.at(a_gate_reshape_m), a_transpose->input_value(1)); + + const ov::OutputVector args = { + hidden, routing, ids, + gate.weight, gate.scale, gate.has_zp ? gate.zp : absent_zp(), + up.weight, up.scale, up.has_zp ? up.zp : absent_zp(), + down.weight, down.scale, down.has_zp ? down.zp : absent_zp(), + }; + + auto moe = std::make_shared<ov::op::internal::MOECompressed>(args, config); + + // MOE takes its output type from the hidden state, which is f16 on GPU, while the rest of + // the ggml graph works in f32. + ov::Output<ov::Node> result = moe->output(0); + const auto root_type = m.get_match_root()->get_output_element_type(0); + if (result.get_element_type() != root_type) { + result = std::make_shared<ov::op::v0::Convert>(result, root_type); + } + + result.get_node_shared_ptr()->set_friendly_name(m.get_match_root()->get_friendly_name()); + ov::copy_runtime_info(m.get_matched_nodes(), result.get_node_shared_ptr()); + ov::replace_node(m.get_match_root(), result.get_node_shared_ptr()); + register_new_node(moe); + return true; + }; + + register_matcher(std::make_shared<Matcher>(root_m, "ov::frontend::ggml::pass::FuseMoeCompressed"), callback); +} + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h new file mode 100644 index 000000000000..5500bed68af8 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h @@ -0,0 +1,19 @@ +#include "openvino/pass/matcher_pass.hpp" + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +// Folds the MoE expert block emitted for MUL_MAT_ID (3 GatherMatmul + SwiGLU + routing +// weighting + expert reduction) into a single ov::op::internal::MOECompressed. +class FuseMoeCompressed : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseMoeCompressed") + FuseMoeCompressed(); +}; + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp new file mode 100644 index 000000000000..04de2d008c72 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp @@ -0,0 +1,114 @@ +#include "kv_state_seq_axis.h" + +#include <memory> +#include <openvino/core/graph_util.hpp> +#include <openvino/op/assign.hpp> +#include <openvino/op/concat.hpp> +#include <openvino/op/constant.hpp> +#include <openvino/op/read_value.hpp> +#include <openvino/op/transpose.hpp> +#include <vector> + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +namespace { + +const std::vector<int64_t> & seq_axis_perm() { + // [1, seq, n_heads_kv, head_size] <-> [1, n_heads_kv, seq, head_size] + static const std::vector<int64_t> perm{0, 2, 1, 3}; + return perm; +} + +// True when the state still has the frontend's stateful KV layout, so the sequence axis +// can be moved: rank 4, batch and both head dims static, and seq the only dynamic dim, +// at dim 1. Any KV head count is fine. With a single head the rewrite is pure metadata +// ([1, seq, 1, head] and [1, 1, seq, head] are the same memory); with several heads it +// also drops the reader-side transpose of the whole accumulated state, which is where +// most of the gain comes from at depth. +bool can_move_seq_axis(const ov::PartialShape & shape) { + return shape.rank().is_static() && shape.rank().get_length() == 4 && shape[0].is_static() && + shape[1].is_dynamic() && shape[2].is_static() && shape[3].is_static(); +} + +std::shared_ptr<ov::op::v0::Concat> match_kv_append(const std::shared_ptr<ov::op::v6::Assign> & assign) { + auto concat = ov::as_type_ptr<ov::op::v0::Concat>(assign->get_input_node_shared_ptr(0)); + if (!concat || concat->get_input_size() != 2 || concat->get_axis() != 1) { + return nullptr; + } + auto read_value = ov::as_type_ptr<ov::op::v6::ReadValue>(concat->get_input_node_shared_ptr(0)); + if (!read_value || read_value->get_variable() != assign->get_variable()) { + return nullptr; + } + if (!can_move_seq_axis(read_value->get_output_partial_shape(0))) { + return nullptr; + } + return concat; +} + +} // namespace + +bool KVStateSeqAxis::run_on_model(const std::shared_ptr<ov::Model> & model) { + std::vector<std::shared_ptr<ov::op::v6::Assign>> assigns; + for (const auto & op : model->get_ops()) { + if (auto assign = ov::as_type_ptr<ov::op::v6::Assign>(op)) { + assigns.push_back(assign); + } + } + + bool changed = false; + for (const auto & assign : assigns) { + auto concat = match_kv_append(assign); + if (!concat) { + continue; + } + auto read_value = ov::as_type_ptr<ov::op::v6::ReadValue>(concat->get_input_node_shared_ptr(0)); + + auto variable = read_value->get_variable(); + auto info = variable->get_info(); + const auto & shape = info.data_shape; + info.data_shape = ov::PartialShape{shape[0], shape[2], shape[1], shape[3]}; + variable->update(info); + read_value->validate_and_infer_types(); + + auto readers = concat->output(0).get_target_inputs(); + + auto new_rows = concat->input_value(1); + auto perm_in = ov::op::v0::Constant::create(ov::element::i64, {4}, seq_axis_perm()); + concat->set_argument(1, std::make_shared<ov::op::v1::Transpose>(new_rows, perm_in)); + concat->set_axis(2); + concat->validate_and_infer_types(); + + // Readers still expect seq at dim 1. A reader that is itself the inverse + // Transpose wanted seq at dim 2 all along, so drop it; give anything else the + // inverse Transpose so its input is unchanged. + for (const auto & reader : readers) { + auto * node = reader.get_node(); + if (ov::is_type<ov::op::v6::Assign>(node)) { + continue; + } + bool dropped = false; + if (auto * transpose = ov::as_type<ov::op::v1::Transpose>(node)) { + auto order = ov::as_type_ptr<ov::op::v0::Constant>(transpose->get_input_node_shared_ptr(1)); + if (order && order->cast_vector<int64_t>() == seq_axis_perm()) { + ov::replace_output_update_name(transpose->output(0), concat->output(0)); + dropped = true; + } + } + if (!dropped) { + auto perm_out = ov::op::v0::Constant::create(ov::element::i64, {4}, seq_axis_perm()); + reader.replace_source_output(std::make_shared<ov::op::v1::Transpose>(concat->output(0), perm_out)); + } + } + changed = true; + } + + return changed; +} + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h new file mode 100644 index 000000000000..579022c45c59 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h @@ -0,0 +1,24 @@ +#include "openvino/pass/pass.hpp" + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +// Moves the sequence axis of the stateful KV cache from dim 1 to dim 2, i.e. from +// [1, seq, n_heads_kv, head_size] to [1, n_heads_kv, seq, head_size], and updates the +// Concat that appends to it. Two wins: the GPU plugin only appends new tokens in place +// when the growing axis is a spatial axis, and the reader no longer has to transpose the +// whole accumulated state every token (that cost grows with context length, so it is the +// larger win at depth for a model with several KV heads). Only rewrites states that still +// match the frontend layout, so it no-ops if that layout ever changes. +class KVStateSeqAxis : public ov::pass::ModelPass { +public: + OPENVINO_MODEL_PASS_RTTI("ov::frontend::ggml::pass::KVStateSeqAxis") + bool run_on_model(const std::shared_ptr<ov::Model> & model) override; +}; + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp b/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp index 20a3a374934b..09c213f3e17c 100644 --- a/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +++ b/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp @@ -2,6 +2,7 @@ #include <openvino/core/graph_util.hpp> #include <openvino/core/rt_info.hpp> +#include <openvino/core/type.hpp> #include <openvino/op/constant.hpp> #include <openvino/op/matmul.hpp> #include <openvino/op/squeeze.hpp> @@ -26,7 +27,7 @@ SqueezeMatmul::SqueezeMatmul() { const auto callback = [=](ov::pass::pattern::Matcher & m) { const auto & pattern_map = m.get_pattern_value_map(); auto matmul_node = - std::dynamic_pointer_cast<ov::op::v0::MatMul>(pattern_map.at(m_matmul).get_node_shared_ptr()); + ov::as_type_ptr<ov::op::v0::MatMul>(pattern_map.at(m_matmul).get_node_shared_ptr()); auto act = pattern_map.at(m_act); auto wei = pattern_map.at(m_wei); auto act_shape = act.get_partial_shape(); diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index df3a72f3286c..e56a4e41d0b5 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -5,7 +5,9 @@ #include "ggml-openvino/openvino/node_context.h" #include "ggml-openvino/openvino/utils.h" #include "input_model.h" +#include "pass/fuse_moe_compressed.h" #include "pass/fuse_to_conv.h" +#include "pass/kv_state_seq_axis.h" #include "pass/mark_decompression_convert_constant_folding.h" #include "pass/mark_dequantization_subgraph.h" #include "pass/squeeze_matmul.h" @@ -19,28 +21,36 @@ #include <openvino/core/node.hpp> #include <openvino/core/preprocess/pre_post_process.hpp> #include <openvino/core/shape.hpp> +#include <openvino/core/type.hpp> #include <openvino/core/type/element_type.hpp> #include <openvino/op/add.hpp> #include <openvino/op/broadcast.hpp> #include <openvino/op/concat.hpp> +#include <openvino/op/constant.hpp> #include <openvino/op/convert.hpp> #include <openvino/op/convert_like.hpp> #include <openvino/op/cos.hpp> #include <openvino/op/divide.hpp> #include <openvino/op/gather.hpp> +#include <openvino/op/greater_eq.hpp> +#include <openvino/op/less.hpp> +#include <openvino/op/logical_and.hpp> #include <openvino/op/multiply.hpp> #include <openvino/op/parameter.hpp> #include <openvino/op/range.hpp> #include <openvino/op/reshape.hpp> #include <openvino/op/result.hpp> +#include <openvino/op/select.hpp> #include <openvino/op/sin.hpp> #include <openvino/op/slice.hpp> #include <openvino/op/squeeze.hpp> #include <openvino/op/strided_slice.hpp> +#include <openvino/op/subtract.hpp> #include <openvino/op/transpose.hpp> #include <openvino/op/unsqueeze.hpp> #include <openvino/pass/constant_folding.hpp> #include <openvino/pass/make_stateful.hpp> +#include <limits> #include <sstream> namespace ov { @@ -143,6 +153,64 @@ void add_sliced_mask_stateful(TensorMap & tensor_map) { create_sliced_mask("self_kq_mask_swa", "KQ_mask_swa_sliced"); } +// Rebuild the sliding-window mask from absolute positions. +// ggml caps self_kq_mask_swa at the size of its own SWA cache, but the stateful KV state is +// Concat-appended and grows without bound, so past that cap the two disagree on length and the +// mask add fails. A pure-Concat state is ordered by position, so positions can rebuild the mask. +// swa_window holds the real n_swa, read back from the ggml mask in ggml-decoder.cpp. +// No-op when the graph has no SWA mask, or when the window could not be read back. +void add_position_mask_stateful_swa(TensorMap & tensor_map) { + if (tensor_map.find("self_kq_mask_swa") == tensor_map.end() || tensor_map.find("inp_pos") == tensor_map.end() || + tensor_map.find("swa_window") == tensor_map.end()) { + return; + } + + auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr(); + + auto zero_i64 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto one_i64 = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + + auto query_pos = std::make_shared<ov::op::v0::Convert>(inp_pos, ov::element::i64); + auto query_pos_1d = std::make_shared<ov::op::v1::Reshape>( + query_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}), false); + + auto last_pos = std::make_shared<ov::op::v8::Gather>(inp_pos, neg_one, three); + auto last_pos_1d = std::make_shared<ov::op::v1::Reshape>(last_pos, one_i64, false); + auto last_pos_cvt = std::make_shared<ov::op::v0::Convert>(last_pos_1d, ov::element::i64); + auto total_len = std::make_shared<ov::op::v1::Add>(last_pos_cvt, one_i64); + auto total_len_scalar = std::make_shared<ov::op::v0::Squeeze>(total_len); + + auto cached_pos = std::make_shared<ov::op::v4::Range>( + ov::op::v0::Constant::create(ov::element::i64, {}, {0}), total_len_scalar, + ov::op::v0::Constant::create(ov::element::i64, {}, {1}), ov::element::i64); + + auto query_col = std::make_shared<ov::op::v1::Reshape>( + query_pos_1d, ov::op::v0::Constant::create(ov::element::i64, {2}, {-1, 1}), false); + auto cached_row = std::make_shared<ov::op::v1::Reshape>( + cached_pos, ov::op::v0::Constant::create(ov::element::i64, {2}, {1, -1}), false); + auto diff = std::make_shared<ov::op::v1::Subtract>(query_col, cached_row); + + auto swa_window = tensor_map.at("swa_window").get_node_shared_ptr(); + auto window = std::make_shared<ov::op::v0::Convert>(swa_window, ov::element::i64); + auto causal_ok = std::make_shared<ov::op::v1::GreaterEqual>(diff, zero_i64); + auto window_ok = std::make_shared<ov::op::v1::Less>(diff, window); + auto keep = std::make_shared<ov::op::v1::LogicalAnd>(causal_ok, window_ok); + + auto zero_f = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f}); + auto neg_inf_f = ov::op::v0::Constant::create(ov::element::f32, {}, {-std::numeric_limits<float>::infinity()}); + std::shared_ptr<ov::Node> mask = std::make_shared<ov::op::v1::Select>(keep, zero_f, neg_inf_f); + + auto batch_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + mask = std::make_shared<ov::op::v0::Unsqueeze>(mask, batch_axis); + mask = std::make_shared<ov::op::v0::Unsqueeze>(mask, batch_axis); + mask = std::make_shared<ov::op::v0::Convert>(mask, ov::element::f16); + mask->set_friendly_name("KQ_mask_swa_sliced"); + + tensor_map["KQ_mask_swa_sliced"] = mask->output(0); +} + void add_rope_sin_cos(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { // When ROPE ops in the graph have divergent op_params (e.g. gemma4's mixed // SWA/non-SWA layers with different n_dims or freq_base), a shared sin/cos @@ -175,6 +243,7 @@ void add_rope_sin_cos(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) void preprocess(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { if (ggml_model_decoder.is_stateful()) { add_sliced_mask_stateful(tensor_map); + add_position_mask_stateful_swa(tensor_map); } // This optimization is error-prone // add_rope_sin_cos(tensor_map, ggml_model_decoder); @@ -204,7 +273,7 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo auto tensor_map = std::make_shared<TensorMap>(); std::shared_ptr<Model> resulting_model; - const auto & ggml_model = std::dynamic_pointer_cast<InputModel>(input_model); + const auto & ggml_model = ov::as_type_ptr<InputModel>(input_model); std::shared_ptr<GgmlDecoder> ggml_model_decoder = ggml_model->get_model_decoder(); for (const auto & it : ggml_model_decoder->get_model_inputs()) { @@ -216,7 +285,7 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo for (const auto & it : ggml_model_decoder->get_model_extra_inputs()) { auto input_node = create_extra_input(it.first, it.second); if (it.second.is_parameter) { - params.push_back(std::dynamic_pointer_cast<ov::op::v0::Parameter>(input_node)); + params.push_back(ov::as_type_ptr<ov::op::v0::Parameter>(input_node)); } (*tensor_map)[it.first] = input_node; } @@ -275,7 +344,7 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo } }; - auto node_visitor = [&](std::shared_ptr<GgmlDecoder> decoder, int node_idx) { + auto node_visitor = [&](const std::shared_ptr<GgmlDecoder> & decoder, int node_idx) { auto converted_outputs = translate_node(decoder, node_idx); if (converted_outputs.empty()) { return; @@ -387,7 +456,7 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo } std::shared_ptr<Model> TranslateSession::apply_transformations(std::shared_ptr<Model> model) { - auto ggml_model_decoder = std::dynamic_pointer_cast<InputModel>(m_input_model)->get_model_decoder(); + auto ggml_model_decoder = ov::as_type_ptr<InputModel>(m_input_model)->get_model_decoder(); { ov::pass::Manager manager; manager.set_per_pass_validation(true); @@ -400,10 +469,20 @@ std::shared_ptr<Model> TranslateSession::apply_transformations(std::shared_ptr<M std::vector<ov::element::Type>{ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4}); manager.register_pass<pass::FuseToConv>(); + // MOECompressed has no CPU plugin implementation, so keep the GatherMatmul path + // everywhere else. Opt-in while the fused path is being brought up. + if (ggml_openvino_get_device_name() == "GPU" && getenv("GGML_OPENVINO_MOE_OP")) { + manager.register_pass<pass::FuseMoeCompressed>(); + } + if (ggml_model_decoder->is_stateful()) { const auto kv_param_res_names = ggml_model_decoder->get_kv_param_res_names(); const auto kv_param_res_pairs = get_kv_param_res_pairs(model, kv_param_res_names); manager.register_pass<ov::pass::MakeStateful>(kv_param_res_pairs); + // Must run after MakeStateful, which is what creates the ReadValue/Assign pairs. + if (!ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT")) { + manager.register_pass<pass::KVStateSeqAxis>(); + } } if (ggml_model_decoder->is_static()) { diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index 8bb7678ee381..98a85e632a9e 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -1,7 +1,5 @@ #include "utils.h" -#include "ggml-impl.h" - #include <cmath> #include <cstddef> #include <ctime> @@ -28,13 +26,6 @@ namespace ov { namespace frontend { namespace ggml { -std::string getCurrentTime() { - std::time_t now = std::time(nullptr); - char buf[100]; - std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", std::localtime(&now)); - return buf; -} - void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs) { auto input_size = context.get_input_size(); FRONT_END_OP_CONVERSION_CHECK(input_size >= min_inputs, "Got less inputs than expected"); @@ -82,7 +73,7 @@ namespace { ov::Output<ov::Node> rope_yarn_ramp_mix(int n_dims, const float corr_dims[2], float ext_factor) { int half_n_dims = n_dims / 2; std::vector<float> dim_ids_vec(half_n_dims); - std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0); + std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0.0f); auto dim_ids = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, (size_t) half_n_dims}, dim_ids_vec); auto corr_low = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[0]}); auto corr_high = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[1]}); @@ -551,6 +542,7 @@ ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int inp if (tail_begin >= 0 && tail_end <= tail_src_elems) { std::vector<int64_t> flat_shape; + flat_shape.reserve(slice_dim); for (int i = 0; i < slice_dim; ++i) { flat_shape.push_back(static_cast<int64_t>(view_src_ggml_shape[i])); } diff --git a/ggml/src/ggml-openvino/openvino/utils.h b/ggml/src/ggml-openvino/openvino/utils.h index 5d4c3538664a..d9858f923655 100644 --- a/ggml/src/ggml-openvino/openvino/utils.h +++ b/ggml/src/ggml-openvino/openvino/utils.h @@ -14,8 +14,6 @@ namespace ggml { std::string getCurrentTime(); -void dump_ov_model(std::shared_ptr<ov::Model> model); - void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs); int non_cont_dim(std::vector<size_t> ne, std::vector<size_t> nb); diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 93b1ccbe9075..b1ee792fdb64 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -2,6 +2,7 @@ #include "ggml-impl.h" #include "ggml-openvino-extra.h" +#include "ggml-openvino.h" #include "ggml-openvino/ggml-decoder.h" #include "ggml.h" #include "model-cache.h" @@ -17,7 +18,6 @@ #include <cstring> #include <fstream> #include <functional> -#include <future> #include <iomanip> #include <iostream> #include <memory> @@ -38,36 +38,7 @@ #include <unordered_map> #include <vector> -// Suppress deprecation warning for ov::Tensor::data() -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - -enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) { - ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context; - try { - if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_CGRAPH")) { - std::string filename = "cgraph_ov.txt"; - GgmlOvDecoder::dump_cgraph(cgraph, filename); - } - - const auto is_static = ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC"); - - GGML_ASSERT(ctx->runtime_context != nullptr); - std::shared_ptr<ov_runtime_context> r_ctx = std::static_pointer_cast<ov_runtime_context>(ctx->runtime_context); - - return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx); - } catch (const ov::Exception & e) { - GGML_LOG_ERROR("GGML OpenVINO backend ov::Exception: %s\n", e.what()); - return GGML_STATUS_FAILED; - } catch (const std::exception & e) { - GGML_LOG_ERROR("GGML OpenVINO backend std::exception: %s\n", e.what()); - return GGML_STATUS_FAILED; - } catch (...) { - GGML_LOG_ERROR("GGML OpenVINO backend unknown exception\n"); - return GGML_STATUS_FAILED; - } -} - +namespace { // For a KV cache input, return an ov::Tensor sized to n_kv (== attention_size // for that layer) instead of the fully-allocated ctx_per_seq. Pre-conditions: // * non-static (CPU/GPU) backend, single sequence, seq_active_start == 0 @@ -78,9 +49,9 @@ enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) // n_kv rows no longer contain the live prefix // On any unmet pre-condition returns std::nullopt; the caller falls back to // the full-size tensor. -static std::optional<ov::Tensor> try_make_kv_sliced_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, - const std::string & name, - const ggml_tensor * ggml_tensor) { +std::optional<ov::Tensor> try_make_kv_sliced_tensor(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder, + const std::string & name, + const ggml_tensor * ggml_tensor) { static const bool kv_slice_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE"); if (kv_slice_disabled) { return std::nullopt; @@ -118,7 +89,7 @@ static std::optional<ov::Tensor> try_make_kv_sliced_tensor(std::shared_ptr<GgmlO return std::nullopt; } - ov::Shape full_shape = ggml_decoder->get_shape(ggml_tensor); + ov::Shape full_shape = GgmlOvDecoder::get_shape(ggml_tensor); if (full_shape.size() != 4 || full_shape[0] != 1 || full_shape[1] != 1 || static_cast<int>(full_shape[2]) != ctx_per_seq) { return std::nullopt; @@ -134,16 +105,16 @@ static std::optional<ov::Tensor> try_make_kv_sliced_tensor(std::shared_ptr<GgmlO // return gpu_context.create_tensor(ggml_decoder->get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data); // } - return ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data); + return ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data); } -static uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device, bool stateful) { +uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device, bool stateful) { const char * manual_gqa_env = ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN"); const bool manual_gqa_enabled = manual_gqa_env != nullptr ? ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0 : device == "GPU"; - uint64_t extra_cfg = 0; + uint64_t extra_cfg = 1; // Graph-ordinal port names (invalidate older disk-cache blobs). extra_cfg = extra_cfg * 131 + (stateful ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_reduce_compile_mem_enabled() ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE") ? 1u : 0u); @@ -151,8 +122,95 @@ static uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device, return extra_cfg; } -ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, - std::shared_ptr<ov::InferRequest> infer_request, +std::map<std::string, std::shared_ptr<ov::Node>> get_weight_names(ggml_cgraph * cgraph) { + std::map<std::string, std::shared_ptr<ov::Node>> names; + for (const auto & name : GgmlOvDecoder::collect_weight_names(cgraph)) { + names[name] = nullptr; + } + return names; +} + +// A conservative, exact in-process key, evaluated only on a context-local cache +// miss. Include topology, layouts, op parameters, constant extra inputs and weight +// allocation identities. Never use a sampled weight hash or a graph name alone: +// different models can have identical topology. OV buffer IDs survive address reuse. +std::string compiled_graph_key(const ggml_cgraph * graph, + const GgmlOvDecoder & decoder, + const std::string & device, + int prefill_chunk_size = 0) { + std::string key; + auto append = [&key](const auto & value) { + key.append(reinterpret_cast<const char *>(&value), sizeof(value)); + }; + auto append_string = [&](const std::string & value) { + append(value.size()); + key.append(value); + }; + append_string(device); + append(decoder.is_static()); + append(decoder.is_stateful()); + append(prefill_chunk_size); + bool has_weight_buffer_id = false; + std::unordered_map<const ggml_tensor *, size_t> ids; + std::function<void(const ggml_tensor *)> visit = [&](const ggml_tensor * tensor) { + if (!tensor) { + append(size_t(0)); + return; + } + auto inserted = ids.emplace(tensor, ids.size() + 1); + append(inserted.first->second); + if (!inserted.second) { + return; + } + append_string(tensor->name); + append(tensor->type); + append(tensor->op); + append(tensor->flags); + append(tensor->ne); + append(tensor->nb); + append(tensor->op_params); + append(tensor->view_offs); + const auto * base = tensor->view_src ? tensor->view_src : tensor; + const bool weight = base->buffer && base->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; + append(weight); + if (weight) { + const size_t buffer_id = ggml_backend_openvino_buffer_get_ctx_id(base->buffer); + has_weight_buffer_id |= buffer_id != 0; + append(buffer_id); + append(tensor->data); + } + visit(tensor->view_src); + for (const auto * src : tensor->src) { + visit(src); + } + }; + append(graph->n_nodes); + for (int i = 0; i < graph->n_nodes; ++i) { + visit(graph->nodes[i]); + } + append(graph->n_leafs); + for (int i = 0; i < graph->n_leafs; ++i) { + visit(graph->leafs[i]); + } + for (const auto & input : decoder.get_model_extra_inputs()) { + append_string(input.first); + append_string(input.second.type.get_type_name()); + append(input.second.shape.size()); + for (auto dim : input.second.shape) { + append(dim); + } + append(input.second.is_parameter); + if (!input.second.is_parameter) { + append(input.second.value); + } + } + // Without an allocation generation, pointer reuse could select stale weights. + // Such graphs still get private requests; they simply do not share compilation. + return has_weight_buffer_id ? key : std::string{}; +} + +ov::Tensor create_ov_output_tensor(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder, + const std::shared_ptr<ov::InferRequest> & infer_request, int output_index, const ggml_tensor * ggml_tensor) { if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, std::string(ggml_tensor->name), ggml_tensor)) { @@ -168,7 +226,7 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, // } // } - auto output_type = ggml_decoder->get_ov_type(ggml_tensor); + auto output_type = GgmlOvDecoder::get_ov_type(ggml_tensor); ov::Shape output_shape; void * output_data = ggml_tensor->data; if (ggml_decoder->is_static()) { @@ -181,199 +239,656 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, // pointer instead so the OV tensor matches the model output exactly. if (ggml_tensor->op == GGML_OP_CPY && ggml_tensor->view_src != nullptr && ggml_nbytes(ggml_tensor) != ggml_nbytes(ggml_tensor->view_src)) { - output_shape = ggml_decoder->get_shape(ggml_tensor->view_src); + output_shape = GgmlOvDecoder::get_shape(ggml_tensor->view_src); output_data = ggml_tensor->view_src->data; } else { - output_shape = ggml_decoder->get_shape(ggml_tensor); + output_shape = GgmlOvDecoder::get_shape(ggml_tensor); } } ov::Tensor output_tensor(output_type, output_shape, output_data); return output_tensor; } -enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx) { - auto & core = ov_singleton_core(); - const auto & config = ggml_openvino_get_compile_config(); - const auto & device = r_ctx->device; - const auto & stateful = r_ctx->stateful; - static auto is_static = false; - - static const bool cache_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); - - // is_model_splitted is O(n_nodes^2) plus a create_weight_nodes scan and takes ~20 ms - // on a Llama-1B decode graph. It is called once per graph_compute invocation but the - // graph shape is identical across all decode steps, so memoize by graph_key: compute - // graph_key first (a few hundred us), and if the same key is already in decoder_cache - // we know the graph is not splitted (only not-splitted graphs get inserted there). - graph_key key(cgraph); - bool key_seen = false; - if (!cache_disabled) { - std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); - key_seen = r_ctx->decoder_cache.find(key) != r_ctx->decoder_cache.end(); +// Rewrite ggml's KV rows into a relayout state that keeps the sequence on dim 2. +// ggml stores [seq][n_heads_kv * head_size]; the state wants [1, n_heads_kv, seq, head_size], +// a different element order, so the rows are copied instead of reinterpreted. +ov::Tensor kv_rows_to_seq_axis_2(const ov::Tensor & kv_tensor, size_t n_heads_kv) { + const size_t rows = kv_tensor.get_shape()[2]; + const size_t head_size = kv_tensor.get_shape()[3] / n_heads_kv; + const size_t elem = kv_tensor.get_element_type().size(); + const size_t head_bytes = head_size * elem; + + ov::Tensor out(kv_tensor.get_element_type(), ov::Shape{1, n_heads_kv, rows, head_size}); + const auto * src = static_cast<const uint8_t *>(kv_tensor.data()); + auto * dst = static_cast<uint8_t *>(out.data()); + for (size_t s = 0; s < rows; s++) { + for (size_t h = 0; h < n_heads_kv; h++) { + memcpy(dst + (h * rows + s) * head_bytes, src + (s * n_heads_kv + h) * head_bytes, head_bytes); + } } + return out; +} - bool model_is_splitted = key_seen ? false : is_model_splitted(cgraph); - - if (is_naive(cgraph)) { - if (!model_is_splitted) { - return naive_compute(cgraph, core, device, config); - } +template <typename T> void set_zero_diagonal(std::vector<T> & matrix, size_t rows, size_t cols, T zero_value = T{}) { + for (size_t i = 0; i < rows; ++i) { + size_t diag_col = std::min(i, cols - 1); + matrix[i * cols + diag_col] = zero_value; } +} - auto start_time = ggml_time_us(); +ov::Tensor make_contiguous_split_input_tensor(const struct ggml_tensor * ggml_tensor, const ov::Shape & input_shape) { + const size_t element_size = ggml_type_size(ggml_tensor->type); + const size_t block_size = ggml_blck_size(ggml_tensor->type); - std::shared_ptr<GgmlOvDecoder> ggml_decoder; - std::shared_ptr<ov::InferRequest> infer_request; - ModelParams m_params; - ComputeParams c_params; - std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static); + GGML_ASSERT(block_size == 1 && "non-contiguous split inputs must be plain element types"); - const bool cache_enabled = !model_is_splitted && !cache_disabled; - bool cache_hit = false; + const struct ggml_tensor * source_tensor = ggml_tensor->view_src != nullptr ? ggml_tensor->view_src : ggml_tensor; + const size_t source_offset = ggml_tensor->view_src != nullptr ? ggml_tensor->view_offs : 0; - int64_t decoder_end_time; - int64_t conversion_end_time; - int64_t compile_end_time; - int64_t infer_end_time; - int64_t ov_raw_infer_start; + std::vector<uint8_t> source_data(ggml_nbytes(source_tensor)); + ggml_backend_tensor_get(source_tensor, source_data.data(), 0, source_data.size()); - { - std::shared_ptr<decoder_runtime_ctx> entry; - ModelParams old_m_params; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + auto * dst = static_cast<uint8_t *>(input_tensor.data()); + size_t dst_offset = 0; - if (cache_enabled) { - std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); - auto it = r_ctx->decoder_cache.find(key); - cache_hit = it != r_ctx->decoder_cache.end(); - if (cache_hit) { - entry = it->second; - } else { - r_ctx->clear_caches_locked(); - auto mutex = std::make_shared<std::mutex>(); - entry = std::make_shared<decoder_runtime_ctx>(mutex); - r_ctx->decoder_cache[key] = entry; + for (size_t i3 = 0; i3 < static_cast<size_t>(ggml_tensor->ne[3]); ++i3) { + for (size_t i2 = 0; i2 < static_cast<size_t>(ggml_tensor->ne[2]); ++i2) { + for (size_t i1 = 0; i1 < static_cast<size_t>(ggml_tensor->ne[1]); ++i1) { + for (size_t i0 = 0; i0 < static_cast<size_t>(ggml_tensor->ne[0]); ++i0) { + const size_t src_offset = source_offset + i3 * ggml_tensor->nb[3] + i2 * ggml_tensor->nb[2] + + i1 * ggml_tensor->nb[1] + i0 * ggml_tensor->nb[0]; + std::memcpy(dst + dst_offset, source_data.data() + src_offset, element_size); + dst_offset += element_size; + } } - } else { - auto mutex = std::make_shared<std::mutex>(); - entry = std::make_shared<decoder_runtime_ctx>(mutex); - cache_hit = false; } + } - std::lock_guard<std::mutex> lock(*(entry->mutex)); + return input_tensor; +} - if (cache_hit) { - ggml_decoder = entry->ptr; - old_m_params = ggml_decoder->get_model_params(); - if (!ggml_decoder->is_splited_model()) { - cache_hit = old_m_params.can_reuse_dynamically(m_params); - } +ov::Tensor convert_ggml_input_to_ov(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder, const std::string & name) { + const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name); + + if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, name, ggml_tensor)) { + return *sliced; + } + + if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) { + auto * extra_base = static_cast<ggml_openvino_extra_base *>(ggml_tensor->extra); + if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) { + // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str()); + auto * tensor_extra = static_cast<ggml_openvino_tensor_extra *>(extra_base); + return *tensor_extra->tensor; } + } - std::vector<std::string> ov_input_names; - std::vector<std::string> ov_output_names; + // GGML_LOG_DEBUG("Converting ggml tensor to ov::Tensor for input: %s\n", name.c_str()); + auto * input_data = ggml_tensor->data; + ov::Shape input_shape; + if (ggml_tensor->op == GGML_OP_VIEW && !ggml_decoder->is_splited_model()) { + // This case is added to make test-backend-ops work + input_shape = GgmlOvDecoder::get_shape(ggml_tensor->view_src); + } else { + input_shape = GgmlOvDecoder::get_shape(ggml_tensor); + } - if (cache_hit) { - std::map<std::string, std::shared_ptr<ov::Node>> model_weights; - ggml_decoder->set_compute_params(c_params); - ggml_decoder->set_model_params(m_params); - if (old_m_params.kv_buffer_changed(m_params)) { - ggml_decoder->update_io(cgraph); - } - ggml_decoder->add_extra_inputs(); - { - std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); - infer_request = r_ctx->infer_request_cache.at(key); - ov_input_names = r_ctx->ov_input_names_cache.at(key); - ov_output_names = r_ctx->ov_output_names_cache.at(key); - } + if (ggml_decoder->is_splited_model() && !ggml_is_contiguous(ggml_tensor)) { + return make_contiguous_split_input_tensor(ggml_tensor, input_shape); + } - if (stateful) { - const auto * inp_pos = get_inp_pos_tensor(cgraph); - int32_t * pos_data = (int32_t *) inp_pos->data; - auto pos_shape = ggml_decoder->get_shape(inp_pos); - if (pos_data[0] == 0) { - infer_request->reset_state(); - r_ctx->stateful_kv_size = pos_shape[3]; - } else if (r_ctx->stateful_kv_size == static_cast<size_t>(pos_data[0])) { - r_ctx->stateful_kv_size += pos_shape[3]; - } else { - auto states = infer_request->query_state(); - for (auto state : states) { - auto state_tensor = state.get_state(); - auto state_tensor_shape = state_tensor.get_shape(); - if (static_cast<uint32_t>(pos_data[0]) > r_ctx->stateful_kv_size) { - std::string state_name; - try { - state_name = r_ctx->kv_state_input_name_map.at(state.get_name()); - } catch (...) { - GGML_LOG_ERROR( - "GGML OpenVINO backend stateful inference failed: no input found for the state\n"); - return GGML_STATUS_FAILED; - } - auto kv_tensor = get_ov_input_tensor(ggml_decoder, state_name); - kv_tensor.set_shape({state_tensor_shape[0], kv_tensor.get_shape()[2], state_tensor_shape[2], - state_tensor_shape[3]}); - state_tensor = kv_tensor; - state_tensor_shape = state_tensor.get_shape(); - } - ov::Coordinate begin = {0, 0, 0, 0}; - ov::Coordinate end = {state_tensor_shape[0], static_cast<uint32_t>(pos_data[0]), - state_tensor_shape[2], state_tensor_shape[3]}; - ov::Tensor new_state_tensor(state_tensor, begin, end); - state.set_state(new_state_tensor); - } - r_ctx->stateful_kv_size = pos_data[0] + pos_shape[3]; - } - } + auto input_tensor = ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape, input_data); + return input_tensor; +} - decoder_end_time = ggml_time_us(); - conversion_end_time = decoder_end_time; - compile_end_time = decoder_end_time; - } else { - // Fail fast: a cache-miss recompile feeds weight data to compile_model, but - // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU) - // may have already dropped the host weight pages - // (they would read as zeros). That mode requires stable graph shapes. - if (ggml_openvino_weight_buffers_released()) { - GGML_ABORT( - "ggml-openvino: a new graph needs to be compiled but host weight buffers were already " - "released via GGML_OPENVINO_RELEASE_WEIGHTS/GGML_OPENVINO_MEMORY_OPTIMIZE. This mode requires " - "stable graph shapes; disable host weight release for dynamic workloads."); - } - if (cache_enabled) { - std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); - r_ctx->infer_request_cache.erase(key); - } +ov::Tensor get_ov_input_tensor(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder, const std::string & param_name) { + ov::Tensor input_tensor; + auto extra_input = ggml_decoder->get_model_extra_inputs().find(param_name); + if (extra_input != ggml_decoder->get_model_extra_inputs().end()) { + input_tensor = ov::Tensor(extra_input->second.type, extra_input->second.shape); + *input_tensor.data<int64_t>() = extra_input->second.value; + } else { + input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name); + } + return input_tensor; +} - // Frontend-level compiled-model cache (GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR): if this model - // was compiled before, import the saved blob and skip requant + convert + - // compile. Only the dynamic single-model path is cached (split models compile - // two graphs and are left to the plugin-level ov::cache_dir). The decoder is - // still needed for I/O mapping, but can be built without weight nodes since - // the weights are baked into the imported CompiledModel. - const std::string model_cache_dir = ggml_openvino_model_cache_dir(); - uint64_t model_fp = 0; - std::string blob_path, manifest_path; - bool imported = false; - // When the frontend model cache is active it supersedes the plugin-level - // ov::cache_dir: a blob exported from a model compiled WITH cache_dir cannot - // be re-imported (import returns an uninitialized model). Strip cache_dir / - // cache_mode from the config used for the cached compile and the import. - ov::AnyMap mc_config = config; - if (!model_cache_dir.empty()) { - mc_config.erase("CACHE_DIR"); - mc_config.erase("CACHE_MODE"); - } - if (!model_cache_dir.empty() && !model_is_splitted) { - const uint64_t extra_cfg = ggml_openvino_model_cache_extra_cfg(device, stateful); - model_fp = ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params, - 15, extra_cfg); - blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp); - manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp); +ov::Tensor get_ov_input_tensor_static_decode(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder, + const std::string & param_name) { + // NPU decoding stage + if (ggml_decoder->get_model_extra_inputs().count(param_name)) { + return get_ov_input_tensor(ggml_decoder, param_name); + } + const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); + const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); + + if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || + GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { + // IMROPE's inp_pos holds one value per t/h/w/e plane instead of a single position; + // with a single decode token the planes are still contiguous, so a flat copy works. + const int n_planes = GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ? GgmlOvDecoder::get_inp_pos_n_planes(op) : 1; + assert(ggml_tensor->ne[0] == n_planes); + ov::Shape input_shape = {1, 1, 1, (size_t) n_planes}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + std::memcpy(input_tensor.data(), ggml_tensor->data, n_planes * ggml_type_size(ggml_tensor->type)); + return input_tensor; + } + + if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { + ov::Shape input_shape = {1, 1, 1, 1}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + int32_t inp_out_id = *((int32_t *) ggml_tensor->data); + assert(ggml_tensor->ne[0] == 1); + assert(inp_out_id == 0); + *input_tensor.data<int32_t>() = inp_out_id; + return input_tensor; + } + + if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { + size_t context_size = ggml_decoder->get_ctx_size(); + if (ggml_tensor->type == GGML_TYPE_F16) { + std::vector<ggml_fp16_t> padded_data = + pad_input<ggml_fp16_t>(ggml_tensor, 1, context_size, GGML_FP32_TO_FP16(-INFINITY)); + ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, 1, context_size}); + std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); + return input_tensor; + } + + std::vector<float> padded_data = pad_input<float>(ggml_tensor, 1, context_size, -INFINITY); + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, 1, context_size}); + auto * data_ptr = input_tensor.data<float>(); + std::copy(padded_data.begin(), padded_data.begin() + context_size, data_ptr); + return input_tensor; + } + + return get_ov_input_tensor(ggml_decoder, param_name); +} + +ov::Tensor get_ov_input_tensor_static_prefill(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder, + const std::string & param_name, + int chunk_index) { + // NPU prompt processing stage + const size_t input_len = ggml_decoder->get_input_len(); + const size_t chunk_size = ggml_decoder->m_prefill_chunk_size; + const size_t chunk_valid_size = std::min(chunk_size, input_len - chunk_index * chunk_size); + const size_t chunk_pad_size = chunk_size - chunk_valid_size; + + if (param_name == "chunk_valid_len") { + ov::Tensor input_tensor(ov::element::i64, ov::Shape{1}); + *input_tensor.data<int64_t>() = (int64_t) chunk_valid_size; + return input_tensor; + } + if (chunk_index > 0 && param_name == "cache_rs_reset_len") { + // The recurrent-state clear belongs to the start of the sequence. Re-applying it on every + // chunk would wipe the state accumulated by the preceding chunks, so disable it (a zero + // length makes scale.cpp's keep-mask select every slot) after the first chunk. + ov::Tensor input_tensor(ov::element::i64, ov::Shape{1}); + *input_tensor.data<int64_t>() = 0; + return input_tensor; + } + if (ggml_decoder->get_model_extra_inputs().count(param_name)) { + return get_ov_input_tensor(ggml_decoder, param_name); + } + const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); + const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); + + if (GgmlOvDecoder::is_inp_pos(ggml_tensor, op) && GgmlOvDecoder::get_inp_pos_n_planes(op) > 1) { + // IMROPE: inp_pos stacks n_planes (t/h/w/e) position planes, each of length + // input_len; pad every plane independently so they stay aligned to chunk_size. + const int n_planes = GgmlOvDecoder::get_inp_pos_n_planes(op); + const size_t element_size = ggml_type_size(ggml_tensor->type); + ov::Shape input_shape = {1, 1, 1, (size_t) n_planes * chunk_size}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + for (int p = 0; p < n_planes; p++) { + const char * src = + (const char *) ggml_tensor->data + (p * input_len + chunk_index * chunk_size) * element_size; + char * dst = (char *) input_tensor.data() + p * chunk_size * element_size; + std::memcpy(dst, src, chunk_valid_size * element_size); + if (chunk_pad_size > 0) { + if (ggml_tensor->type == GGML_TYPE_I32) { + int32_t last_value = *((const int32_t *) src + chunk_valid_size - 1); + int32_t * out = (int32_t *) dst; + std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1); + } else if (ggml_tensor->type == GGML_TYPE_I64) { + int64_t last_value = *((const int64_t *) src + chunk_valid_size - 1); + int64_t * out = (int64_t *) dst; + std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1); + } else { + throw std::runtime_error("Unexpected tensor type for " + param_name); + } + } + } + return input_tensor; + } + + if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || + GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { + ov::Shape input_shape = {1, 1, 1, chunk_size}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + // copy the chunk_index-th chunk from ggml_tensor + size_t element_size = ggml_type_size(ggml_tensor->type); + void * input_data = (char *) ggml_tensor->data + chunk_index * chunk_size * element_size; + std::memcpy(input_tensor.data(), input_data, chunk_valid_size * element_size); + // pad the rest with last_value + 1, so that kv's of padded positions are inserted + // to the next row after the valids row in the kvcache + if (chunk_pad_size > 0) { + if (ggml_tensor->type == GGML_TYPE_I32) { + int32_t last_value = + *((int32_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); + int32_t * output_data = input_tensor.data<int32_t>(); + std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); + } else if (ggml_tensor->type == GGML_TYPE_I64) { + int64_t last_value = + *((int64_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); + int64_t * output_data = input_tensor.data<int64_t>(); + std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); + } else { + throw std::runtime_error("Unexpected tensor type for " + param_name); + } + } + return input_tensor; + } + + if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { + size_t output_len = ggml_decoder->get_compute_params().output_len; + ov::Shape input_shape = {1, 1, 1, output_len}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + if (ggml_tensor->ne[0] == 0) { + *input_tensor.data<int32_t>() = 0; + } else { + auto * data_addr = input_tensor.data<int32_t>(); + for (size_t i = 0; i < output_len; i++) { + data_addr[i] = ((int32_t *) ggml_tensor->data)[i] % chunk_size; + } + } + return input_tensor; + } + + if (GgmlOvDecoder::is_inp_mean(ggml_tensor, op)) { + const size_t n_seqs = ggml_tensor->ne[1]; + const size_t src_stride = ggml_tensor->ne[0]; + const size_t copy_len = std::min<size_t>(chunk_valid_size, src_stride - chunk_index * chunk_size); + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, n_seqs, chunk_size}); + auto * dst = input_tensor.data<float>(); + std::fill(dst, dst + n_seqs * chunk_size, 0.0f); + const auto * src = static_cast<const float *>(ggml_tensor->data) + chunk_index * chunk_size; + for (size_t s = 0; s < n_seqs; s++) { + std::memcpy(dst + s * chunk_size, src + s * src_stride, copy_len * sizeof(float)); + } + return input_tensor; + } + + if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { + size_t cols = ggml_tensor->ne[0]; + size_t rows = ggml_tensor->ne[1]; + size_t chunk_valid_rows = std::min(chunk_size, rows - chunk_index * chunk_size); + size_t context_size = ggml_decoder->get_ctx_size(); + if (ggml_tensor->type == GGML_TYPE_F16) { + const auto * ggml_data = + static_cast<const ggml_fp16_t *>(ggml_tensor->data) + chunk_index * chunk_size * cols; + std::vector<ggml_fp16_t> padded_data = pad_input<ggml_fp16_t>(ggml_data, chunk_valid_rows, cols, chunk_size, + context_size, GGML_FP32_TO_FP16(-INFINITY)); + set_zero_diagonal(padded_data, chunk_size, context_size, GGML_FP32_TO_FP16(0.0f)); + ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, chunk_size, context_size}); + std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); + return input_tensor; + } + + const auto * ggml_data = static_cast<const float *>(ggml_tensor->data) + chunk_index * chunk_size * cols; + std::vector<float> padded_data = + pad_input<float>(ggml_data, chunk_valid_rows, cols, chunk_size, context_size, -INFINITY); + set_zero_diagonal(padded_data, chunk_size, context_size); + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, chunk_size, context_size}); + auto * data_ptr = input_tensor.data<float>(); + std::copy(padded_data.begin(), padded_data.begin() + chunk_size * context_size, data_ptr); + return input_tensor; + } + + return get_ov_input_tensor(ggml_decoder, param_name); +} + +enum ggml_status naive_compute(ggml_cgraph * cgraph, + ov::Core & core, + const std::string & device, + const ov::AnyMap & config, + ov_compiled_model_cache & cache) { + if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) { + return GGML_STATUS_SUCCESS; + } + + std::unique_lock<std::mutex> compile_lock(cache.mutex); + bool naive = true; + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); + auto decoder = std::make_shared<GgmlOvDecoder>(cgraph, model_weights); + auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(decoder); + auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { + ov::serialize(model, "IR_naive.xml"); + } + + std::shared_ptr<ov::InferRequest> infer_request; + auto remote_context = ggml_openvino_get_remote_context(); + ov::AnyMap compile_config = config; + if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) { + // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved + compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::PERFORMANCE; + } else { + compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::ACCURACY; + } + if (remote_context.has_value()) { + infer_request = std::make_shared<ov::InferRequest>( + core.compile_model(model, remote_context.value(), compile_config).create_infer_request()); + } else { + infer_request = std::make_shared<ov::InferRequest>( + core.compile_model(model, device, compile_config).create_infer_request()); + } + std::vector<std::string> input_names; + std::vector<std::string> output_names; + for (const auto & param : model->get_parameters()) { + input_names.push_back(param->get_friendly_name()); + } + for (const auto & result : model->get_results()) { + output_names.push_back(result->get_friendly_name()); + } + // Destroy the frontend graph under the compilation lock as well: it can + // still own edges into the shared weight nodes. + model.reset(); + input_model.reset(); + decoder->clear_model_weights(); + model_weights.clear(); + compile_lock.unlock(); + + for (size_t i = 0; i < input_names.size(); i++) { + const auto & param_name = input_names[i]; + auto input_tensor = get_ov_input_tensor(decoder, param_name); + infer_request->set_input_tensor(i, input_tensor); + } + + // Use get_output_tensor + memcpy instead of set_output_tensor to avoid memory overwritten + // when i/o buffer overlaps, e.g. the cgraph is a single PERMUTE + + infer_request->infer(); + + for (size_t i = 0; i < output_names.size(); i++) { + auto output_tensor = infer_request->get_output_tensor(i); + const auto & model_outputs = decoder->get_model_outputs(); + auto model_output_it = model_outputs.find(output_names[i]); + if (model_output_it == model_outputs.end()) { + // Debug-only output added via GGML_OPENVINO_DEBUG_NODE; nothing to copy into. + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || + ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + print_output_tensor_info(output_names[i], output_tensor, output_tensor.data()); + } + continue; + } + auto * ggml_tensor = model_output_it->second; + std::memcpy(ggml_tensor->data, output_tensor.data(), output_tensor.get_byte_size()); + } + return GGML_STATUS_SUCCESS; +} + +enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::shared_ptr<ov_runtime_context> & r_ctx) { + auto & core = ov_singleton_core(); + const auto & config = ggml_openvino_get_compile_config(); + const auto & device = r_ctx->device; + const auto & stateful = r_ctx->stateful; + static auto is_static = false; + + static const bool cache_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); + + // is_model_splitted is O(n_nodes^2) plus a create_weight_nodes scan and takes ~20 ms + // on a Llama-1B decode graph. It is called once per graph_compute invocation but the + // graph shape is identical across all decode steps, so memoize by graph_key: compute + // graph_key first (a few hundred us), and if the same key is already in decoder_cache + // we know the graph is not splitted (only not-splitted graphs get inserted there). + graph_key key(cgraph); + bool key_seen = false; + if (!cache_disabled) { + std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); + key_seen = r_ctx->decoder_cache.find(key) != r_ctx->decoder_cache.end(); + } + + bool model_is_splitted = key_seen ? false : is_model_splitted(cgraph); + + if (is_naive(cgraph)) { + if (!model_is_splitted) { + return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache); + } + } + + auto start_time = ggml_time_us(); + + std::shared_ptr<GgmlOvDecoder> ggml_decoder; + std::shared_ptr<ov::InferRequest> infer_request; + ModelParams m_params; + ComputeParams c_params; + std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static); + + const bool cache_enabled = !model_is_splitted && !cache_disabled; + bool cache_hit = false; + + int64_t decoder_end_time; + int64_t conversion_end_time; + int64_t compile_end_time; + int64_t infer_end_time; + int64_t ov_raw_infer_start; + + { + std::shared_ptr<decoder_runtime_ctx> entry; + ModelParams old_m_params; + + if (cache_enabled) { + std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); + auto it = r_ctx->decoder_cache.find(key); + cache_hit = it != r_ctx->decoder_cache.end(); + if (cache_hit) { + entry = it->second; + } else { + r_ctx->clear_caches_locked(); + auto mutex = std::make_shared<std::mutex>(); + entry = std::make_shared<decoder_runtime_ctx>(mutex); + r_ctx->decoder_cache[key] = entry; + } + } else { + auto mutex = std::make_shared<std::mutex>(); + entry = std::make_shared<decoder_runtime_ctx>(mutex); + cache_hit = false; + } + + std::lock_guard<std::mutex> lock(*(entry->mutex)); + cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0; + + if (cache_hit) { + ggml_decoder = entry->ptr; + old_m_params = ggml_decoder->get_model_params(); + if (!ggml_decoder->is_splited_model()) { + cache_hit = old_m_params.can_reuse_dynamically(m_params); + } + } + + std::vector<std::string> ov_input_names; + std::vector<std::string> ov_output_names; + + if (cache_hit) { + std::map<std::string, std::shared_ptr<ov::Node>> model_weights; + ggml_decoder->set_compute_params(c_params); + ggml_decoder->set_model_params(m_params); + if (old_m_params.kv_buffer_changed(m_params)) { + ggml_decoder->update_io(cgraph); + } + ggml_decoder->add_extra_inputs(); + { + std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); + infer_request = r_ctx->infer_request_cache.at(key); + ov_input_names = r_ctx->ov_input_names_cache.at(key); + ov_output_names = r_ctx->ov_output_names_cache.at(key); + } + + if (stateful) { + const auto * inp_pos = get_inp_pos_tensor(cgraph); + int32_t * pos_data = (int32_t *) inp_pos->data; + auto pos_shape = GgmlOvDecoder::get_shape(inp_pos); + if (pos_data[0] == 0) { + infer_request->reset_state(); + r_ctx->stateful_kv_size = pos_shape[3]; + } else if (r_ctx->stateful_kv_size == static_cast<size_t>(pos_data[0])) { + r_ctx->stateful_kv_size += pos_shape[3]; + } else { + const size_t pos_begin = static_cast<size_t>(pos_data[0]); + const bool refill = pos_begin > r_ctx->stateful_kv_size; + + // A refill seeds the state from ggml's KV cache, so it needs that cache to be a + // plain prefix: cell i must hold position i. An SWA layer keeps only the last + // n_swa positions, so once a position leaves the window ggml drops it and the + // remaining cells shift - cell i stops holding position i. While every position + // is still inside the window nothing has been dropped and the refill is sound. + if (refill && !ggml_decoder->get_model_params().swa_layers.empty()) { + const int n_swa = ggml_decoder->get_compute_params().swa_window; + if (n_swa < 0 || static_cast<size_t>(n_swa) < pos_begin) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: cannot resume at position %zu from a " + "state that holds %zu tokens, because the sliding-window layers keep only the last %d " + "positions. Run without GGML_OPENVINO_STATEFUL_EXECUTION.\n", + pos_begin, r_ctx->stateful_kv_size, n_swa); + return GGML_STATUS_FAILED; + } + } + + const bool relayout_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT"); + + auto states = infer_request->query_state(); + for (auto state : states) { + auto state_tensor = state.get_state(); + auto state_tensor_shape = state_tensor.get_shape(); + + std::string state_name; + if (auto it = r_ctx->kv_state_input_name_map.find(state.get_name()); + it != r_ctx->kv_state_input_name_map.end()) { + state_name = it->second; + } + + // Which axis holds the sequence: pass::KVStateSeqAxis moves it from dim 1 + // to dim 2. The head count is still needed below, because only a 1-head + // state stays byte-compatible with ggml's cache buffer. gemma-4 12B mixes + // 1-head full layers with 8-head sliding layers, so it is per state. + int n_heads_kv = ggml_decoder->get_model_params().n_heads_kv; + if (auto layer = extract_layer_from_name(state_name); layer.has_value()) { + n_heads_kv = ggml_decoder->get_n_heads_kv_for_layer(layer.value()); + } + const bool relayout_this_state = relayout_enabled; + const size_t seq_axis = relayout_this_state ? 2 : 1; + const size_t head_axis = seq_axis == 2 ? 1 : 2; + + if (refill) { + if (state_name.empty()) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: no input found for the state\n"); + return GGML_STATUS_FAILED; + } + auto kv_tensor = get_ov_input_tensor(ggml_decoder, state_name); + if (relayout_this_state && n_heads_kv != 1) { + // several heads with seq on dim 2: not the same bytes as ggml's + // buffer, so the rows have to be copied into the new order + state_tensor = kv_rows_to_seq_axis_2(kv_tensor, (size_t) n_heads_kv); + } else { + ov::Shape refill_shape(4); + refill_shape[0] = state_tensor_shape[0]; + refill_shape[seq_axis] = kv_tensor.get_shape()[2]; + refill_shape[head_axis] = state_tensor_shape[head_axis]; + refill_shape[3] = state_tensor_shape[3]; + kv_tensor.set_shape(refill_shape); + state_tensor = kv_tensor; + } + state_tensor_shape = state_tensor.get_shape(); + } + // Only ever shrink to a prefix the source really has. Slicing past it used to + // surface as a bare ov::Exception from the ROI constructor. + if (state_tensor_shape[seq_axis] < pos_begin) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: state '%s' holds %zu tokens on axis " + "%zu, cannot resume at position %zu\n", + state.get_name().c_str(), state_tensor_shape[seq_axis], seq_axis, pos_begin); + return GGML_STATUS_FAILED; + } + ov::Coordinate begin = {0, 0, 0, 0}; + ov::Coordinate end(state_tensor_shape.begin(), state_tensor_shape.end()); + end[seq_axis] = pos_begin; + ov::Tensor new_state_tensor(state_tensor, begin, end); + state.set_state(new_state_tensor); + } + r_ctx->stateful_kv_size = pos_begin + pos_shape[3]; + } + } + + decoder_end_time = ggml_time_us(); + conversion_end_time = decoder_end_time; + compile_end_time = decoder_end_time; + } else { + // Compilation can mutate shared weight nodes, so serialize cold paths. + // The lock is released before binding tensors or running inference. + auto shared_cache = r_ctx->compiled_cache; + std::unique_lock<std::mutex> compile_lock(shared_cache->mutex); + auto weight_names = get_weight_names(cgraph); + ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, weight_names, is_static, + stateful, model_is_splitted); + const std::string shared_key = cache_enabled ? compiled_graph_key(cgraph, *ggml_decoder, device) : ""; + ov::CompiledModel shared_model; + bool imported = false; + auto shared_it = shared_cache->graphs.find(shared_key); + if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) { + shared_model = shared_it->second.decode; + infer_request = std::make_shared<ov::InferRequest>(shared_model.create_infer_request()); + ov_input_names = shared_it->second.input_names; + ov_output_names = shared_it->second.output_names; + imported = true; + GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (dynamic)\n"); + } + // Fail fast: a cache-miss recompile feeds weight data to compile_model, but + // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU) + // may have already dropped the host weight pages + // (they would read as zeros). That mode requires stable graph shapes. + if (!imported && ggml_openvino_weight_buffers_released()) { + GGML_ABORT( + "ggml-openvino: a new graph needs to be compiled but host weight buffers were already " + "released via GGML_OPENVINO_RELEASE_WEIGHTS/GGML_OPENVINO_MEMORY_OPTIMIZE. This mode requires " + "stable graph shapes; disable host weight release for dynamic workloads."); + } + if (cache_enabled) { + std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache.erase(key); + } + + // Frontend-level compiled-model cache (GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR): if this model + // was compiled before, import the saved blob and skip requant + convert + + // compile. Only the dynamic single-model path is cached (split models compile + // two graphs and are left to the plugin-level ov::cache_dir). The decoder is + // still needed for I/O mapping, but can be built without weight nodes since + // the weights are baked into the imported CompiledModel. + const std::string model_cache_dir = ggml_openvino_model_cache_dir(); + uint64_t model_fp = 0; + std::string blob_path; + std::string manifest_path; + // When the frontend model cache is active it supersedes the plugin-level + // ov::cache_dir: a blob exported from a model compiled WITH cache_dir cannot + // be re-imported (import returns an uninitialized model). Strip cache_dir / + // cache_mode from the config used for the cached compile and the import. + ov::AnyMap mc_config = config; + if (!model_cache_dir.empty()) { + mc_config.erase("CACHE_DIR"); + mc_config.erase("CACHE_MODE"); + } + if (!imported && !model_cache_dir.empty() && !model_is_splitted) { + const uint64_t extra_cfg = ggml_openvino_model_cache_extra_cfg(device, stateful); + model_fp = + ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params, 16, extra_cfg); + blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp); + manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp); std::ifstream blob_in(blob_path, std::ios::binary); bool blob_ok = blob_in.is_open(); - bool manifest_ok = blob_ok && ggml_openvino_model_cache_verify_manifest(manifest_path, cgraph, model_fp); + bool manifest_ok = + blob_ok && ggml_openvino_model_cache_verify_manifest(manifest_path, cgraph, model_fp); if (blob_ok && manifest_ok) { int64_t import_start = ggml_time_us(); try { @@ -393,6 +908,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, weight_names, is_static, stateful, model_is_splitted); infer_request = std::make_shared<ov::InferRequest>(cm.create_infer_request()); + shared_model = cm; entry->ptr = ggml_decoder; // Names must match the decoder's ggml-tensor keys. The non-cached // path keys off Parameter/Result *friendly names* (set by the @@ -425,412 +941,131 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< } else { auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); - ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static, - stateful, model_is_splitted); - decoder_end_time = ggml_time_us(); - - auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(ggml_decoder); - model = ov::frontend::ggml::FrontEnd::convert(input_model); - ggml_decoder->clear_model_weights(); - conversion_end_time = ggml_time_us(); - - if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { - char timestamped_filename[64]; - auto timestamp = (long long) ggml_time_us(); - snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%lld.xml", timestamp); - ov::serialize(model, timestamped_filename); - } - - // Use the cache-stripped config when the frontend model cache is active, so - // the resulting CompiledModel can be exported and later re-imported. - const ov::AnyMap & compile_config = model_cache_dir.empty() ? config : mc_config; - ov::CompiledModel compiled_model; - auto remote_context = ggml_openvino_get_remote_context(); - if (remote_context.has_value()) { - compiled_model = core.compile_model(model, remote_context.value(), compile_config); - } else { - compiled_model = core.compile_model(model, device, compile_config); - } - compile_end_time = ggml_time_us(); - - // Export to the frontend model cache for next time. Publish the blob first, - // then the manifest, so a cache hit only sees fully written artifacts. - if (!model_cache_dir.empty() && !model_is_splitted && model_fp != 0) { - try { - const std::string blob_tmp = blob_path + ".tmp"; - const std::string manifest_tmp = manifest_path + ".tmp"; - if (ggml_openvino_model_cache_write_manifest(manifest_tmp, cgraph, model_fp)) { - std::ofstream blob_out(blob_tmp, std::ios::binary | std::ios::trunc); - if (blob_out.is_open()) { - compiled_model.export_model(blob_out); - blob_out.close(); - if (blob_out.good()) { - if (std::rename(blob_tmp.c_str(), blob_path.c_str()) == 0 && - std::rename(manifest_tmp.c_str(), manifest_path.c_str()) == 0) { - GGML_LOG_INFO("ggml-openvino: model cache WROTE %s\n", blob_path.c_str()); - } else { - std::remove(blob_tmp.c_str()); - std::remove(manifest_tmp.c_str()); - } - } else { - std::remove(blob_tmp.c_str()); - std::remove(manifest_tmp.c_str()); - } - } else { - std::remove(manifest_tmp.c_str()); - } - } - } catch (const std::exception & e) { - GGML_LOG_WARN("ggml-openvino: model cache export failed: %s\n", e.what()); - } - } - - infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request()); - entry->ptr = ggml_decoder; - - for (const auto & ov_param : model->get_parameters()) { - ov_input_names.push_back(ov_param->get_friendly_name()); - } - for (const auto & ov_output : model->get_results()) { - ov_output_names.push_back(ov_output->get_friendly_name()); - } - } // end non-imported (compile) path - - if (cache_enabled) { - std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); - r_ctx->infer_request_cache[key] = infer_request; - r_ctx->ov_input_names_cache[key] = ov_input_names; - r_ctx->ov_output_names_cache[key] = ov_output_names; - } - - if (stateful && cache_enabled) { - const auto * inp_pos = get_inp_pos_tensor(cgraph); - auto pos_shape = ggml_decoder->get_shape(inp_pos); - r_ctx->stateful_kv_size = pos_shape[3]; - const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names(); - for (const auto & pair : kv_param_res_names) { - r_ctx->kv_state_input_name_map[pair.first + pair.second] = pair.first; - } - } - } - - for (size_t i = 0; i < ov_input_names.size(); i++) { - auto param_name = ov_input_names[i]; - auto input_tensor = get_ov_input_tensor(ggml_decoder, param_name); - infer_request->set_input_tensor(i, input_tensor); - - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { - print_input_tensor_info(param_name, input_tensor); - } - } - - for (size_t i = 0; i < ov_output_names.size(); i++) { - // Debug-only outputs added via GGML_OPENVINO_DEBUG_NODE (see - // translate_session.cpp) have no corresponding ggml tensor; leave - // them unbound so OpenVINO allocates its own tensor for them, - // rather than aliasing a ggml buffer that may be overwritten by a - // later in-place op before we get to read it. - const auto & model_outputs = ggml_decoder->get_model_outputs(); - auto model_output_it = model_outputs.find(ov_output_names[i]); - if (model_output_it == model_outputs.end()) { - continue; - } - auto * ggml_tensor = model_output_it->second; - if (ggml_nbytes(ggml_tensor) == 0) { - continue; - } - auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); - infer_request->set_output_tensor(i, output_tensor); - } - - ov_raw_infer_start = ggml_time_us(); - infer_request->infer(); - infer_end_time = ggml_time_us(); - - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || - ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { - for (size_t i = 0; i < ov_output_names.size(); i++) { - const auto output_tensor = infer_request->get_output_tensor(i); - print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data()); - } - } - - if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { - GGML_LOG_INFO("\nGGML OpenVINO Backend: \n"); - GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0); - if (!cache_hit) { - GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", - (conversion_end_time - decoder_end_time) / 1000.0); - GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0); - } - GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0); - GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", (infer_end_time - ov_raw_infer_start) / 1000.0); - } - } - - // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU): the plugin holds its own device copy of - // every weight after compile, so the host weight buffers can be dropped to reclaim - // RSS. The GPU backend uses a single dynamic-shape model for both prefill and decode, - // so once a graph is compiled it is reused for the whole session — the only thing - // that forces a recompile is clear_caches() on backend teardown. We therefore release - // on the first cache-hit (model compiled, plugin has its copy) and, crucially, pin the - // compiled-model cache so it survives backend teardown (see ggml_backend_openvino_free). - // Without the pin, a later test/context would recompile against the now-dropped pages. - // A genuinely new graph still fails fast at the cache-miss compile branch. - if (cache_hit && ggml_openvino_release_weights_enabled(device) && - !ggml_openvino_weight_buffers_released()) { - ggml_openvino_release_weight_buffers(); - } - - return GGML_STATUS_SUCCESS; -} - -enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx) { - auto & core = ov_singleton_core(); - - auto get_prefill_chunk_size = [] { - static const int chunk_size = []() { - int env_prefill_chunk_size = ggml_openvino_getenv_int("GGML_OPENVINO_PREFILL_CHUNK_SIZE"); - return env_prefill_chunk_size > 0 ? env_prefill_chunk_size : 256; - }(); - return chunk_size; - }; - - // Normally NPU, but honors GGML_OPENVINO_DEVICE so GGML_OPENVINO_FORCE_STATIC can run the - // static-shape path on CPU/GPU to isolate translation bugs from NPUW/NPU-driver issues. - static std::string device = ggml_openvino_get_device_name(); - static auto is_static = true; - static auto stateful = false; - - auto prefill_chunk_size = get_prefill_chunk_size(); - const auto & config = ggml_openvino_get_compile_config(); - - if (is_naive(cgraph)) { - return naive_compute(cgraph, core, device, config); - } - - auto start_time = ggml_time_us(); - - std::shared_ptr<GgmlOvDecoder> ggml_decoder; - std::shared_ptr<ov::InferRequest> infer_request; - ModelParams m_params; - ComputeParams c_params; - std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static); - - const auto * inp_pos = get_inp_pos_tensor(cgraph); - const auto is_prefill = get_is_prefill(cgraph, inp_pos); - graph_key key(cgraph); - static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); - bool cache_hit = false; - - int64_t decoder_end_time; - int64_t conversion_end_time; - int64_t compile_end_time; - int64_t infer_end_time; - int64_t ov_raw_infer_start; - int64_t ov_raw_infer_total = 0; - - std::shared_ptr<decoder_runtime_ctx> entry; - ModelParams old_m_params; - - if (cache_enabled) { - std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); - auto it = r_ctx->decoder_cache.find(key); - cache_hit = it != r_ctx->decoder_cache.end(); - if (cache_hit) { - entry = it->second; - } else { - r_ctx->clear_caches_locked(); - auto mutex = std::make_shared<std::mutex>(); - entry = std::make_shared<decoder_runtime_ctx>(mutex); - r_ctx->decoder_cache[key] = entry; - } - } else { - auto mutex = std::make_shared<std::mutex>(); - entry = std::make_shared<decoder_runtime_ctx>(mutex); - cache_hit = false; - } - - std::lock_guard<std::mutex> lock(*(entry->mutex)); - - if (cache_hit) { - ggml_decoder = entry->ptr; - old_m_params = ggml_decoder->get_model_params(); - cache_hit = old_m_params.can_reuse_statically(m_params); - } - - std::vector<std::string> ov_input_names_local; - std::vector<std::string> ov_output_names_local; - - if (cache_hit) { - std::map<std::string, std::shared_ptr<ov::Node>> model_weights; - ggml_decoder->m_is_prefill = is_prefill; - ggml_decoder->set_model_params(m_params); - ggml_decoder->set_compute_params(c_params); - if (old_m_params.kv_buffer_changed(m_params)) { - ggml_decoder->update_io(cgraph); - } - ggml_decoder->add_extra_inputs(); - { - std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); - infer_request = - is_prefill ? r_ctx->infer_request_cache_prefill.at(key) : r_ctx->infer_request_cache.at(key); - ov_input_names_local = r_ctx->ov_input_names_cache.at(key); - ov_output_names_local = r_ctx->ov_output_names_cache.at(key); - } - - decoder_end_time = ggml_time_us(); - conversion_end_time = decoder_end_time; - compile_end_time = decoder_end_time; - } else { - if (cache_enabled) { - std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); - r_ctx->infer_request_cache.erase(key); - r_ctx->infer_request_cache_prefill.erase(key); - } - - std::shared_ptr<ov::Model> model; - auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); - - if (m_params.n_heads_kv == -1) { - // graph is not a LLM, e.g. context-shift graph - prefill_chunk_size = inp_pos->ne[0]; - } - auto ggml_decoder_prefill = std::make_shared<GgmlOvDecoder>( - cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size); - auto ggml_decoder_decode = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static, - stateful, false, false, prefill_chunk_size); - decoder_end_time = ggml_time_us(); - - const bool dump_ir = ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR"); - const auto dump_ir_timestamp = static_cast<long long>(ggml_time_us()); - - auto build_static_model = [&core, &config, dump_ir, dump_ir_timestamp]( - std::shared_ptr<GgmlOvDecoder> decoder, - const char * tag, - std::shared_ptr<ov::Model> & model, - ov::CompiledModel & compiled_model, - std::shared_ptr<ov::InferRequest> & infer_request, - int64_t & local_conversion_end_time, - int64_t & local_compile_end_time) { - auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(decoder); - model = ov::frontend::ggml::FrontEnd::convert(input_model); - decoder->clear_model_weights(); - local_conversion_end_time = ggml_time_us(); - - if (dump_ir) { - char timestamped_filename[64]; - snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%s_%lld.xml", tag, - dump_ir_timestamp); - ov::serialize(model, timestamped_filename); - } + ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static, + stateful, model_is_splitted); + decoder_end_time = ggml_time_us(); - compiled_model = core.compile_model(model, device, config); - infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request()); - local_compile_end_time = ggml_time_us(); - }; - std::shared_ptr<ov::Model> model_prefill; - std::shared_ptr<ov::Model> model_decode; - ov::CompiledModel compiled_model_prefill; - ov::CompiledModel compiled_model_decode; - std::shared_ptr<ov::InferRequest> infer_request_prefill; - std::shared_ptr<ov::InferRequest> infer_request_decode; - int64_t prefill_conversion_end_time; - int64_t decode_conversion_end_time; - int64_t prefill_compile_end_time; - int64_t decode_compile_end_time; - auto prefill_future = std::async(std::launch::async, build_static_model, ggml_decoder_prefill, "prefill", - std::ref(model_prefill), std::ref(compiled_model_prefill), - std::ref(infer_request_prefill), std::ref(prefill_conversion_end_time), - std::ref(prefill_compile_end_time)); - auto decode_future = std::async(std::launch::async, build_static_model, ggml_decoder_decode, "decode", - std::ref(model_decode), std::ref(compiled_model_decode), - std::ref(infer_request_decode), std::ref(decode_conversion_end_time), - std::ref(decode_compile_end_time)); - prefill_future.get(); - decode_future.get(); - conversion_end_time = std::max(prefill_conversion_end_time, decode_conversion_end_time); - compile_end_time = std::max(prefill_compile_end_time, decode_compile_end_time); - - model = is_prefill ? model_prefill : model_decode; - ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode; - infer_request = is_prefill ? infer_request_prefill : infer_request_decode; - entry->ptr = ggml_decoder; - - for (const auto & ov_param : model->get_parameters()) { - ov_input_names_local.push_back(ov_param->get_friendly_name()); - } - for (const auto & ov_output : model->get_results()) { - ov_output_names_local.push_back(ov_output->get_friendly_name()); - } + auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(ggml_decoder); + model = ov::frontend::ggml::FrontEnd::convert(input_model); + ggml_decoder->clear_model_weights(); + conversion_end_time = ggml_time_us(); - if (cache_enabled) { - std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); - r_ctx->infer_request_cache_prefill[key] = infer_request_prefill; - r_ctx->infer_request_cache[key] = infer_request_decode; - r_ctx->ov_input_names_cache[key] = ov_input_names_local; - r_ctx->ov_output_names_cache[key] = ov_output_names_local; - } - } + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { + char timestamped_filename[64]; + auto timestamp = (long long) ggml_time_us(); + snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%lld.xml", timestamp); + ov::serialize(model, timestamped_filename); + } - if (is_prefill) { - auto inp_len = get_inp_pos_n_tokens(cgraph, inp_pos); - for (int chunk_index = 0; chunk_index * prefill_chunk_size < inp_len; chunk_index++) { - for (size_t i = 0; i < ov_input_names_local.size(); i++) { - auto param_name = ov_input_names_local[i]; - auto input_tensor = get_ov_input_tensor_static_prefill(ggml_decoder, param_name, chunk_index); - infer_request->set_input_tensor(i, input_tensor); + // Use the cache-stripped config when the frontend model cache is active, so + // the resulting CompiledModel can be exported and later re-imported. + const ov::AnyMap & compile_config = model_cache_dir.empty() ? config : mc_config; + ov::CompiledModel compiled_model; + auto remote_context = ggml_openvino_get_remote_context(); + if (remote_context.has_value()) { + compiled_model = core.compile_model(model, remote_context.value(), compile_config); + } else { + compiled_model = core.compile_model(model, device, compile_config); + } + compile_end_time = ggml_time_us(); - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { - const auto input_tensor = infer_request->get_input_tensor(i); - print_input_tensor_info(param_name, input_tensor); + // Export to the frontend model cache for next time. Publish the blob first, + // then the manifest, so a cache hit only sees fully written artifacts. + if (!model_cache_dir.empty() && !model_is_splitted && model_fp != 0) { + try { + const std::string blob_tmp = blob_path + ".tmp"; + const std::string manifest_tmp = manifest_path + ".tmp"; + if (ggml_openvino_model_cache_write_manifest(manifest_tmp, cgraph, model_fp)) { + std::ofstream blob_out(blob_tmp, std::ios::binary | std::ios::trunc); + if (blob_out.is_open()) { + compiled_model.export_model(blob_out); + blob_out.close(); + if (blob_out.good()) { + if (std::rename(blob_tmp.c_str(), blob_path.c_str()) == 0 && + std::rename(manifest_tmp.c_str(), manifest_path.c_str()) == 0) { + GGML_LOG_INFO("ggml-openvino: model cache WROTE %s\n", blob_path.c_str()); + } else { + std::remove(blob_tmp.c_str()); + std::remove(manifest_tmp.c_str()); + } + } else { + std::remove(blob_tmp.c_str()); + std::remove(manifest_tmp.c_str()); + } + } else { + std::remove(manifest_tmp.c_str()); + } + } + } catch (const std::exception & e) { + GGML_LOG_WARN("ggml-openvino: model cache export failed: %s\n", e.what()); + } } - } - for (size_t i = 0; i < ov_output_names_local.size(); i++) { - const auto & model_outputs = ggml_decoder->get_model_outputs(); - auto model_output_it = model_outputs.find(ov_output_names_local[i]); - if (model_output_it == model_outputs.end()) { - continue; + infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request()); + shared_model = compiled_model; + entry->ptr = ggml_decoder; + + for (const auto & ov_param : model->get_parameters()) { + ov_input_names.push_back(ov_param->get_friendly_name()); } - auto * ggml_tensor = model_output_it->second; - if (ggml_nbytes(ggml_tensor) == 0) { - // Zero-row in-place writeback (e.g. the empty s_copy defrag remainder). The OV - // Result is the full cache, so binding it over this 0-byte buffer overflows it. - continue; + for (const auto & ov_output : model->get_results()) { + ov_output_names.push_back(ov_output->get_friendly_name()); } - auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); - infer_request->set_output_tensor(i, output_tensor); - } + } // end non-imported (compile) path - ov_raw_infer_start = ggml_time_us(); - infer_request->infer(); - ov_raw_infer_total += ggml_time_us() - ov_raw_infer_start; + entry->ptr = ggml_decoder; + if (!shared_key.empty() && shared_it == shared_cache->graphs.end()) { + shared_cache->graphs.emplace(shared_key, + ov_compiled_graph{shared_model, {}, ov_input_names, ov_output_names}); + } + if (cache_enabled) { + std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache[key] = infer_request; + r_ctx->ov_input_names_cache[key] = ov_input_names; + r_ctx->ov_output_names_cache[key] = ov_output_names; + } - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || - ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { - for (size_t i = 0; i < ov_output_names_local.size(); i++) { - const auto output_tensor = infer_request->get_output_tensor(i); - print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); + if (stateful && cache_enabled) { + const auto * inp_pos = get_inp_pos_tensor(cgraph); + auto pos_shape = GgmlOvDecoder::get_shape(inp_pos); + // A freshly compiled model starts with an empty state, so it can only serve a + // sequence from its beginning. A non-zero start position means the KV history was + // built elsewhere (a restored ggml cache), which the state cannot adopt. + const int32_t pos_begin = ((int32_t *) inp_pos->data)[0]; + if (pos_begin != 0) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: a new model was compiled for a sequence that " + "starts at position %d, but its state is empty. Run without " + "GGML_OPENVINO_STATEFUL_EXECUTION.\n", + pos_begin); + return GGML_STATUS_FAILED; + } + r_ctx->stateful_kv_size = pos_shape[3]; + const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names(); + for (const auto & pair : kv_param_res_names) { + r_ctx->kv_state_input_name_map[pair.first + pair.second] = pair.first; } } } - infer_end_time = ggml_time_us(); - } else { - for (size_t i = 0; i < ov_input_names_local.size(); i++) { - auto param_name = ov_input_names_local[i]; - auto input_tensor = get_ov_input_tensor_static_decode(ggml_decoder, param_name); + + for (size_t i = 0; i < ov_input_names.size(); i++) { + const auto & param_name = ov_input_names[i]; + auto input_tensor = get_ov_input_tensor(ggml_decoder, param_name); infer_request->set_input_tensor(i, input_tensor); if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { - const auto input_tensor = infer_request->get_input_tensor(i); print_input_tensor_info(param_name, input_tensor); } } - for (size_t i = 0; i < ov_output_names_local.size(); i++) { + for (size_t i = 0; i < ov_output_names.size(); i++) { + // Debug-only outputs added via GGML_OPENVINO_DEBUG_NODE (see + // translate_session.cpp) have no corresponding ggml tensor; leave + // them unbound so OpenVINO allocates its own tensor for them, + // rather than aliasing a ggml buffer that may be overwritten by a + // later in-place op before we get to read it. const auto & model_outputs = ggml_decoder->get_model_outputs(); - auto model_output_it = model_outputs.find(ov_output_names_local[i]); + auto model_output_it = model_outputs.find(ov_output_names[i]); if (model_output_it == model_outputs.end()) { continue; } @@ -845,450 +1080,498 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o ov_raw_infer_start = ggml_time_us(); infer_request->infer(); infer_end_time = ggml_time_us(); - ov_raw_infer_total = infer_end_time - ov_raw_infer_start; if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { - for (size_t i = 0; i < ov_output_names_local.size(); i++) { + for (size_t i = 0; i < ov_output_names.size(); i++) { const auto output_tensor = infer_request->get_output_tensor(i); - print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); + print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data()); + } + } + + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { + GGML_LOG_INFO("\nGGML OpenVINO Backend: \n"); + GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0); + if (!cache_hit) { + GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", + (conversion_end_time - decoder_end_time) / 1000.0); + GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0); } + GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0); + GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", (infer_end_time - ov_raw_infer_start) / 1000.0); } } - if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { - GGML_LOG_INFO("\nGGML OpenVINO Backend: \n"); - GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0); - if (!cache_hit) { - GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", (conversion_end_time - decoder_end_time) / 1000.0); - GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0); + // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU): the plugin holds its own device copy of + // every weight after compile, so the host weight buffers can be dropped to reclaim + // RSS. Release only while holding the compilation mutex so another context cannot + // be reading host weights during conversion/compilation. Pin the shared compiled + // models across backend teardown; a later context can create its own request without + // reading the dropped pages. A new, uncached graph still fails fast above. + if (cache_hit && ggml_openvino_release_weights_enabled(device)) { + std::lock_guard<std::mutex> compile_lock(r_ctx->compiled_cache->mutex); + if (!ggml_openvino_weight_buffers_released()) { + ggml_openvino_release_weight_buffers(); } - GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0); - GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", ov_raw_infer_total / 1000.0); } return GGML_STATUS_SUCCESS; } -// Detect whether a cgraph is a split subgraph or not. -// Step 1 compares each node's recorded use_count with actual fan-out references in node->src. -// Step 2 verifies that node inputs come from model nodes/weights/leafs; external sources imply split. -bool is_model_splitted(ggml_cgraph * cgraph) { - static const bool fallback_enabled = ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_FALLBACK") != 0; - if (!fallback_enabled) { - return false; - } - - // Backend op tests execute each node through ggml_graph_view(), which preserves the original - // graph use_counts while exposing only one node. Treat those single-node views as regular - // naive graphs so intermediate ops do not look like split-model fragments. - if (cgraph->n_nodes <= 1 && cgraph->n_leafs == 0) { - return false; - } - - // check the nodes of the model are used by the following nodes, through compare the node's use count and the count of nodes that use it as input. If does not match, return true, else return false. - for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; - int use_count = cgraph->use_counts[ggml_hash_find(&cgraph->visited_hash_set, node)]; - // TODO: this is a workround for the tests case from llama.cpp, fix should from the root cause in the future. - if ((cgraph->n_nodes <= 1 && use_count == 0) || - (cgraph->n_nodes <= 1 && node->op == GGML_OP_VIEW && use_count == 1 && node->src[0] != nullptr && - node->src[0]->op == GGML_OP_NONE)) { - return false; - } - if (cgraph->n_nodes == 1 && - (cgraph->nodes[0]->op == GGML_OP_TRANSPOSE || cgraph->nodes[0]->op == GGML_OP_PERMUTE)) { - return false; - } - int input_use_count = 0; - for (int j = 0; j < cgraph->n_nodes; j++) { - ggml_tensor * other_node = cgraph->nodes[j]; - for (int k = 0; k < GGML_MAX_SRC; k++) { - if (other_node->src[k] == node) { - input_use_count++; - } - } - } - if (use_count != input_use_count && node->op != GGML_OP_NONE) { - return true; - } - } - // if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check. - // Only weight-name membership is needed below. With GGML_OPENVINO_REDUCE_COMPILE_MEM - // use the name-only collector (no weight extraction); otherwise keep the original - // behavior of building (naive) weight nodes and take their names. - std::set<std::string> model_weights; - if (ggml_openvino_reduce_compile_mem_enabled()) { - model_weights = GgmlOvDecoder::collect_weight_names(cgraph); - } else { - for (const auto & kv : GgmlOvDecoder::create_weight_nodes(cgraph, true)) { - model_weights.insert(kv.first); - } - } - std::set<ggml_tensor *> model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes); - // leaf nodes - std::set<ggml_tensor *> model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs); - for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; - for (int j = 0; j < GGML_MAX_SRC; j++) { - ggml_tensor * src = node->src[j]; - // the src is also not the model weights, we think the model is splitted. - // the src is also not in model leafs, we think the model is splitted. - if (src != nullptr && model_nodes.find(src) == model_nodes.end() && - model_weights.find(std::string(src->name)) == model_weights.end() && !model_leafs.empty() == false && - model_leafs.find(src) == model_leafs.end()) { - if (GgmlOvDecoder::is_inp_tok(src, node)) { - return false; - } - return true; - } +ov::AnyMap without_npuw(const ov::AnyMap & config) { + ov::AnyMap out; + for (const auto & kv : config) { + if (kv.first.rfind("NPUW", 0) == 0 || kv.first == "NPU_USE_NPUW") { + continue; } + out.insert(kv); } - return false; + return out; } -bool is_naive(ggml_cgraph * cgraph) { - constexpr int naive_graph_size_threshold = 20; - int count = 0; - for (int i = 0; i < cgraph->n_nodes; i++) { - if (cgraph->nodes[i]->op != GGML_OP_NONE) { - count++; - } - } - return count < naive_graph_size_threshold; -} +enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared_ptr<ov_runtime_context> & r_ctx) { + auto & core = ov_singleton_core(); + + auto get_prefill_chunk_size = [] { + static const int chunk_size = []() { + int env_prefill_chunk_size = ggml_openvino_getenv_int("GGML_OPENVINO_PREFILL_CHUNK_SIZE"); + return env_prefill_chunk_size > 0 ? env_prefill_chunk_size : 256; + }(); + return chunk_size; + }; + + // Normally NPU, but honors GGML_OPENVINO_DEVICE so GGML_OPENVINO_FORCE_STATIC can run the + // static-shape path on CPU/GPU to isolate translation bugs from NPUW/NPU-driver issues. + static std::string device = ggml_openvino_get_device_name(); + static auto is_static = true; + static auto stateful = false; -enum ggml_status naive_compute(ggml_cgraph * cgraph, - ov::Core & core, - const std::string & device, - const ov::AnyMap & config) { - if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) { - return GGML_STATUS_SUCCESS; - } + auto prefill_chunk_size = get_prefill_chunk_size(); + const auto & config = ggml_openvino_get_compile_config(); - bool naive = true; - auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); - auto decoder = std::make_shared<GgmlOvDecoder>(cgraph, model_weights); - auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(decoder); - auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); - if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { - ov::serialize(model, "IR_naive.xml"); + if (is_naive(cgraph)) { + return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache); } + auto start_time = ggml_time_us(); + + std::shared_ptr<GgmlOvDecoder> ggml_decoder; std::shared_ptr<ov::InferRequest> infer_request; - auto remote_context = ggml_openvino_get_remote_context(); - if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) { - // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved - core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::PERFORMANCE)); - } else { - core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::ACCURACY)); - } - if (remote_context.has_value()) { - infer_request = std::make_shared<ov::InferRequest>( - core.compile_model(model, remote_context.value(), config).create_infer_request()); - } else { - infer_request = - std::make_shared<ov::InferRequest>(core.compile_model(model, device, config).create_infer_request()); - } + ModelParams m_params; + ComputeParams c_params; + std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static); - auto ov_params = model->get_parameters(); - for (size_t i = 0; i < ov_params.size(); i++) { - auto param_name = ov_params[i]->get_friendly_name(); - auto input_tensor = get_ov_input_tensor(decoder, param_name); - infer_request->set_input_tensor(i, input_tensor); + const auto * inp_pos = get_inp_pos_tensor(cgraph); + const bool no_kv_cache = m_params.is_cacheless_attn; + const auto is_prefill = no_kv_cache ? true : get_is_prefill(cgraph, inp_pos); + const ov::AnyMap compile_config = no_kv_cache ? without_npuw(config) : config; + if (m_params.n_heads_kv == -1) { + prefill_chunk_size = inp_pos->ne[0]; } + graph_key key(cgraph); + static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); + bool cache_hit = false; - // Use get_output_tensor + memcpy instead of set_output_tensor to avoid memory overwritten - // when i/o buffer overlaps, e.g. the cgraph is a single PERMUTE + int64_t decoder_end_time; + int64_t conversion_end_time; + int64_t compile_end_time; + int64_t infer_end_time; + int64_t ov_raw_infer_start; + int64_t ov_raw_infer_total = 0; - infer_request->infer(); + std::shared_ptr<decoder_runtime_ctx> entry; + ModelParams old_m_params; - auto ov_results = model->get_results(); - for (size_t i = 0; i < ov_results.size(); i++) { - auto output_tensor = infer_request->get_output_tensor(i); - const auto & model_outputs = decoder->get_model_outputs(); - auto model_output_it = model_outputs.find(ov_results[i]->get_friendly_name()); - if (model_output_it == model_outputs.end()) { - // Debug-only output added via GGML_OPENVINO_DEBUG_NODE; nothing to copy into. - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || - ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { - print_output_tensor_info(ov_results[i]->get_friendly_name(), output_tensor, output_tensor.data()); - } - continue; + if (cache_enabled) { + std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); + auto it = r_ctx->decoder_cache.find(key); + cache_hit = it != r_ctx->decoder_cache.end(); + if (cache_hit) { + entry = it->second; + } else { + r_ctx->clear_caches_locked(); + auto mutex = std::make_shared<std::mutex>(); + entry = std::make_shared<decoder_runtime_ctx>(mutex); + r_ctx->decoder_cache[key] = entry; } - auto * ggml_tensor = model_output_it->second; - std::memcpy(ggml_tensor->data, output_tensor.data(), output_tensor.get_byte_size()); + } else { + auto mutex = std::make_shared<std::mutex>(); + entry = std::make_shared<decoder_runtime_ctx>(mutex); + cache_hit = false; } - return GGML_STATUS_SUCCESS; -} -namespace { -template <typename T> void set_zero_diagonal(std::vector<T> & matrix, size_t rows, size_t cols, T zero_value = T{}) { - for (size_t i = 0; i < rows; ++i) { - size_t diag_col = std::min(i, cols - 1); - matrix[i * cols + diag_col] = zero_value; + std::lock_guard<std::mutex> lock(*(entry->mutex)); + cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0 && + r_ctx->infer_request_cache_prefill.count(key) != 0; + + if (cache_hit) { + ggml_decoder = entry->ptr; + old_m_params = ggml_decoder->get_model_params(); + cache_hit = old_m_params.can_reuse_statically(m_params); } -} -ov::Tensor make_contiguous_split_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, - const struct ggml_tensor * ggml_tensor, - const ov::Shape & input_shape) { - const size_t element_size = ggml_type_size(ggml_tensor->type); - const size_t block_size = ggml_blck_size(ggml_tensor->type); + std::vector<std::string> ov_input_names_local; + std::vector<std::string> ov_output_names_local; - GGML_ASSERT(block_size == 1 && "non-contiguous split inputs must be plain element types"); + if (cache_hit) { + std::map<std::string, std::shared_ptr<ov::Node>> model_weights; + ggml_decoder->m_is_prefill = is_prefill; + ggml_decoder->set_model_params(m_params); + ggml_decoder->set_compute_params(c_params); + if (old_m_params.kv_buffer_changed(m_params)) { + ggml_decoder->update_io(cgraph); + } + ggml_decoder->add_extra_inputs(); + { + std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); + infer_request = + is_prefill ? r_ctx->infer_request_cache_prefill.at(key) : r_ctx->infer_request_cache.at(key); + ov_input_names_local = r_ctx->ov_input_names_cache.at(key); + ov_output_names_local = r_ctx->ov_output_names_cache.at(key); + } - const struct ggml_tensor * source_tensor = ggml_tensor->view_src != nullptr ? ggml_tensor->view_src : ggml_tensor; - const size_t source_offset = ggml_tensor->view_src != nullptr ? ggml_tensor->view_offs : 0; + decoder_end_time = ggml_time_us(); + conversion_end_time = decoder_end_time; + compile_end_time = decoder_end_time; + } else { + if (cache_enabled) { + std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache.erase(key); + r_ctx->infer_request_cache_prefill.erase(key); + } - std::vector<uint8_t> source_data(ggml_nbytes(source_tensor)); - ggml_backend_tensor_get(source_tensor, source_data.data(), 0, source_data.size()); + // Static execution shares a compiled prefill/decode pair. Each backend + // creates and retains its own requests for both phases. + auto shared_cache = r_ctx->compiled_cache; + std::unique_lock<std::mutex> compile_lock(shared_cache->mutex); + auto weight_names = get_weight_names(cgraph); + auto local_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, weight_names, is_static, + stateful, false, is_prefill, prefill_chunk_size); + const std::string shared_key = + cache_enabled ? compiled_graph_key(cgraph, *local_decoder, device, prefill_chunk_size) : ""; + auto shared_it = shared_cache->graphs.find(shared_key); + if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) { + auto & compiled = shared_it->second; + auto prefill_request = std::make_shared<ov::InferRequest>(compiled.prefill.create_infer_request()); + auto decode_request = no_kv_cache ? + prefill_request : + std::make_shared<ov::InferRequest>(compiled.decode.create_infer_request()); + ggml_decoder = local_decoder; + entry->ptr = ggml_decoder; + infer_request = is_prefill ? prefill_request : decode_request; + ov_input_names_local = compiled.input_names; + ov_output_names_local = compiled.output_names; + r_ctx->infer_request_cache_prefill[key] = prefill_request; + r_ctx->infer_request_cache[key] = decode_request; + r_ctx->ov_input_names_cache[key] = ov_input_names_local; + r_ctx->ov_output_names_cache[key] = ov_output_names_local; + decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us(); + GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (static)\n"); + } else { + std::shared_ptr<ov::Model> model; + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); + + auto ggml_decoder_prefill = std::make_shared<GgmlOvDecoder>( + cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size); + auto ggml_decoder_decode = + no_kv_cache ? ggml_decoder_prefill : + std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static, + stateful, false, false, prefill_chunk_size); + decoder_end_time = ggml_time_us(); - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - auto * dst = static_cast<uint8_t *>(input_tensor.data()); - size_t dst_offset = 0; + const bool dump_ir = ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR"); + const auto dump_ir_timestamp = static_cast<long long>(ggml_time_us()); - for (size_t i3 = 0; i3 < static_cast<size_t>(ggml_tensor->ne[3]); ++i3) { - for (size_t i2 = 0; i2 < static_cast<size_t>(ggml_tensor->ne[2]); ++i2) { - for (size_t i1 = 0; i1 < static_cast<size_t>(ggml_tensor->ne[1]); ++i1) { - for (size_t i0 = 0; i0 < static_cast<size_t>(ggml_tensor->ne[0]); ++i0) { - const size_t src_offset = source_offset + i3 * ggml_tensor->nb[3] + i2 * ggml_tensor->nb[2] + - i1 * ggml_tensor->nb[1] + i0 * ggml_tensor->nb[0]; - std::memcpy(dst + dst_offset, source_data.data() + src_offset, element_size); - dst_offset += element_size; + auto build_static_model = [&core, &compile_config, dump_ir, dump_ir_timestamp]( + const std::shared_ptr<GgmlOvDecoder> & decoder, const char * tag, + std::shared_ptr<ov::Model> & model, ov::CompiledModel & compiled_model, + std::shared_ptr<ov::InferRequest> & infer_request, + int64_t & local_conversion_end_time, int64_t & local_compile_end_time) { + auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(decoder); + model = ov::frontend::ggml::FrontEnd::convert(input_model); + decoder->clear_model_weights(); + local_conversion_end_time = ggml_time_us(); + + if (dump_ir) { + char timestamped_filename[64]; + snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%s_%lld.xml", tag, + dump_ir_timestamp); + ov::serialize(model, timestamped_filename); } + + compiled_model = core.compile_model(model, device, compile_config); + infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request()); + local_compile_end_time = ggml_time_us(); + }; + std::shared_ptr<ov::Model> model_prefill; + std::shared_ptr<ov::Model> model_decode; + ov::CompiledModel compiled_model_prefill; + ov::CompiledModel compiled_model_decode; + std::shared_ptr<ov::InferRequest> infer_request_prefill; + std::shared_ptr<ov::InferRequest> infer_request_decode; + int64_t prefill_conversion_end_time; + int64_t decode_conversion_end_time; + int64_t prefill_compile_end_time; + int64_t decode_compile_end_time; + build_static_model(ggml_decoder_prefill, "prefill", model_prefill, compiled_model_prefill, + infer_request_prefill, prefill_conversion_end_time, prefill_compile_end_time); + if (no_kv_cache) { + model_decode = model_prefill; + compiled_model_decode = compiled_model_prefill; + infer_request_decode = infer_request_prefill; + decode_conversion_end_time = prefill_conversion_end_time; + decode_compile_end_time = prefill_compile_end_time; + } else { + build_static_model(ggml_decoder_decode, "decode", model_decode, compiled_model_decode, + infer_request_decode, decode_conversion_end_time, decode_compile_end_time); } - } - } + conversion_end_time = std::max(prefill_conversion_end_time, decode_conversion_end_time); + compile_end_time = std::max(prefill_compile_end_time, decode_compile_end_time); - return input_tensor; -} + model = is_prefill ? model_prefill : model_decode; + ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode; + infer_request = is_prefill ? infer_request_prefill : infer_request_decode; + entry->ptr = ggml_decoder; -ov::Tensor convert_ggml_input_to_ov(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & name) { - const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name); + for (const auto & ov_param : model->get_parameters()) { + ov_input_names_local.push_back(ov_param->get_friendly_name()); + } + for (const auto & ov_output : model->get_results()) { + ov_output_names_local.push_back(ov_output->get_friendly_name()); + } - if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, name, ggml_tensor)) { - return *sliced; - } + if (!shared_key.empty()) { + shared_cache->graphs.emplace( + shared_key, ov_compiled_graph{compiled_model_decode, compiled_model_prefill, ov_input_names_local, + ov_output_names_local}); + } - if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) { - auto * extra_base = static_cast<ggml_openvino_extra_base *>(ggml_tensor->extra); - if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) { - // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str()); - auto * tensor_extra = static_cast<ggml_openvino_tensor_extra *>(extra_base); - return *tensor_extra->tensor; + if (cache_enabled) { + std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache_prefill[key] = infer_request_prefill; + r_ctx->infer_request_cache[key] = infer_request_decode; + r_ctx->ov_input_names_cache[key] = ov_input_names_local; + r_ctx->ov_output_names_cache[key] = ov_output_names_local; + } } } - // GGML_LOG_DEBUG("Converting ggml tensor to ov::Tensor for input: %s\n", name.c_str()); - auto * input_data = ggml_tensor->data; - ov::Shape input_shape; - if (ggml_tensor->op == GGML_OP_VIEW && !ggml_decoder->is_splited_model()) { - // This case is added to make test-backend-ops work - input_shape = ggml_decoder->get_shape(ggml_tensor->view_src); - } else { - input_shape = ggml_decoder->get_shape(ggml_tensor); - } + if (is_prefill) { + auto inp_len = get_inp_pos_n_tokens(cgraph, inp_pos); + for (int chunk_index = 0; chunk_index * prefill_chunk_size < inp_len; chunk_index++) { + for (size_t i = 0; i < ov_input_names_local.size(); i++) { + const auto & param_name = ov_input_names_local[i]; + auto input_tensor = get_ov_input_tensor_static_prefill(ggml_decoder, param_name, chunk_index); + infer_request->set_input_tensor(i, input_tensor); - if (ggml_decoder->is_splited_model() && !ggml_is_contiguous(ggml_tensor)) { - return make_contiguous_split_input_tensor(ggml_decoder, ggml_tensor, input_shape); - } + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { + const auto input_tensor = infer_request->get_input_tensor(i); + print_input_tensor_info(param_name, input_tensor); + } + } - auto input_tensor = ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape, input_data); - return input_tensor; -} -} // namespace + for (size_t i = 0; i < ov_output_names_local.size(); i++) { + const auto & model_outputs = ggml_decoder->get_model_outputs(); + auto model_output_it = model_outputs.find(ov_output_names_local[i]); + if (model_output_it == model_outputs.end()) { + continue; + } + auto * ggml_tensor = model_output_it->second; + if (ggml_nbytes(ggml_tensor) == 0) { + // Zero-row in-place writeback (e.g. the empty s_copy defrag remainder). The OV + // Result is the full cache, so binding it over this 0-byte buffer overflows it. + continue; + } + auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); + infer_request->set_output_tensor(i, output_tensor); + } -ov::Tensor get_ov_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & param_name) { - ov::Tensor input_tensor; - auto extra_input = ggml_decoder->get_model_extra_inputs().find(param_name); - if (extra_input != ggml_decoder->get_model_extra_inputs().end()) { - input_tensor = ov::Tensor(extra_input->second.type, extra_input->second.shape); - *input_tensor.data<int64_t>() = extra_input->second.value; + ov_raw_infer_start = ggml_time_us(); + infer_request->infer(); + ov_raw_infer_total += ggml_time_us() - ov_raw_infer_start; + + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || + ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + for (size_t i = 0; i < ov_output_names_local.size(); i++) { + const auto output_tensor = infer_request->get_output_tensor(i); + print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); + } + } + } + infer_end_time = ggml_time_us(); } else { - input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name); - } - return input_tensor; -} + for (size_t i = 0; i < ov_input_names_local.size(); i++) { + const auto & param_name = ov_input_names_local[i]; + auto input_tensor = get_ov_input_tensor_static_decode(ggml_decoder, param_name); + infer_request->set_input_tensor(i, input_tensor); -ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr<GgmlOvDecoder> ggml_decoder, - const std::string & param_name) { - // NPU decoding stage - if (ggml_decoder->get_model_extra_inputs().count(param_name)) { - return get_ov_input_tensor(ggml_decoder, param_name); - } - const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); - const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { + const auto input_tensor = infer_request->get_input_tensor(i); + print_input_tensor_info(param_name, input_tensor); + } + } - if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || - GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { - // IMROPE's inp_pos holds one value per t/h/w/e plane instead of a single position; - // with a single decode token the planes are still contiguous, so a flat copy works. - const int n_planes = GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ? GgmlOvDecoder::get_inp_pos_n_planes(op) : 1; - assert(ggml_tensor->ne[0] == n_planes); - ov::Shape input_shape = {1, 1, 1, (size_t) n_planes}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - std::memcpy(input_tensor.data(), ggml_tensor->data, n_planes * ggml_type_size(ggml_tensor->type)); - return input_tensor; - } + for (size_t i = 0; i < ov_output_names_local.size(); i++) { + const auto & model_outputs = ggml_decoder->get_model_outputs(); + auto model_output_it = model_outputs.find(ov_output_names_local[i]); + if (model_output_it == model_outputs.end()) { + continue; + } + auto * ggml_tensor = model_output_it->second; + if (ggml_nbytes(ggml_tensor) == 0) { + continue; + } + auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); + infer_request->set_output_tensor(i, output_tensor); + } - if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { - ov::Shape input_shape = {1, 1, 1, 1}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - int32_t inp_out_id = *((int32_t *) ggml_tensor->data); - assert(ggml_tensor->ne[0] == 1); - assert(inp_out_id == 0); - *input_tensor.data<int32_t>() = inp_out_id; - return input_tensor; - } + ov_raw_infer_start = ggml_time_us(); + infer_request->infer(); + infer_end_time = ggml_time_us(); + ov_raw_infer_total = infer_end_time - ov_raw_infer_start; - if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { - size_t context_size = ggml_decoder->get_ctx_size(); - if (ggml_tensor->type == GGML_TYPE_F16) { - std::vector<ggml_fp16_t> padded_data = - pad_input<ggml_fp16_t>(ggml_tensor, 1, context_size, GGML_FP32_TO_FP16(-INFINITY)); - ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, 1, context_size}); - std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); - return input_tensor; + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || + ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + for (size_t i = 0; i < ov_output_names_local.size(); i++) { + const auto output_tensor = infer_request->get_output_tensor(i); + print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); + } } + } - std::vector<float> padded_data = pad_input<float>(ggml_tensor, 1, context_size, -INFINITY); - ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, 1, context_size}); - auto * data_ptr = input_tensor.data<float>(); - std::copy(padded_data.begin(), padded_data.begin() + context_size, data_ptr); - return input_tensor; + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { + GGML_LOG_INFO("\nGGML OpenVINO Backend: \n"); + GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0); + if (!cache_hit) { + GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", (conversion_end_time - decoder_end_time) / 1000.0); + GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0); + } + GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0); + GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", ov_raw_infer_total / 1000.0); } - return get_ov_input_tensor(ggml_decoder, param_name); + return GGML_STATUS_SUCCESS; } +} // namespace -ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr<GgmlOvDecoder> ggml_decoder, - const std::string & param_name, - int chunk_index) { - // NPU prompt processing stage - const size_t input_len = ggml_decoder->get_input_len(); - const size_t chunk_size = ggml_decoder->m_prefill_chunk_size; - const size_t chunk_valid_size = std::min(chunk_size, input_len - chunk_index * chunk_size); - const size_t chunk_pad_size = chunk_size - chunk_valid_size; +// Both execution paths use two cache levels: +// 1. Reuse this backend's decoder/request via graph_key and compatibility checks. +// 2. On a local miss, look up compiled_graph_key in the shared compilation cache, +// compile if needed, then create a private request from the compiled model. +// The shared lock covers compilation and frontend cleanup, never inference. +enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) { + ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context; + try { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_CGRAPH")) { + std::string filename = "cgraph_ov.txt"; + GgmlOvDecoder::dump_cgraph(cgraph, filename); + } - if (param_name == "chunk_valid_len") { - ov::Tensor input_tensor(ov::element::i64, ov::Shape{1}); - *input_tensor.data<int64_t>() = (int64_t) chunk_valid_size; - return input_tensor; + const auto is_static = ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC"); + + GGML_ASSERT(ctx->runtime_context != nullptr); + std::shared_ptr<ov_runtime_context> r_ctx = std::static_pointer_cast<ov_runtime_context>(ctx->runtime_context); + std::lock_guard<std::mutex> execution_lock(r_ctx->execution_mutex); + + return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx); + } catch (const ov::Exception & e) { + GGML_LOG_ERROR("GGML OpenVINO backend ov::Exception: %s\n", e.what()); + return GGML_STATUS_FAILED; + } catch (const std::exception & e) { + GGML_LOG_ERROR("GGML OpenVINO backend std::exception: %s\n", e.what()); + return GGML_STATUS_FAILED; + } catch (...) { + GGML_LOG_ERROR("GGML OpenVINO backend unknown exception\n"); + return GGML_STATUS_FAILED; } - if (chunk_index > 0 && param_name == "cache_rs_reset_len") { - // The recurrent-state clear belongs to the start of the sequence. Re-applying it on every - // chunk would wipe the state accumulated by the preceding chunks, so disable it (a zero - // length makes scale.cpp's keep-mask select every slot) after the first chunk. - ov::Tensor input_tensor(ov::element::i64, ov::Shape{1}); - *input_tensor.data<int64_t>() = 0; - return input_tensor; +} + +// Detect whether a cgraph is a split subgraph or not. +// Step 1 compares each node's recorded use_count with actual fan-out references in node->src. +// Step 2 verifies that node inputs come from model nodes/weights/leafs; external sources imply split. +bool is_model_splitted(ggml_cgraph * cgraph) { + static const bool fallback_enabled = ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_FALLBACK") != 0; + if (!fallback_enabled) { + return false; } - if (ggml_decoder->get_model_extra_inputs().count(param_name)) { - return get_ov_input_tensor(ggml_decoder, param_name); + + // Backend op tests execute each node through ggml_graph_view(), which preserves the original + // graph use_counts while exposing only one node. Treat those single-node views as regular + // naive graphs so intermediate ops do not look like split-model fragments. + if (cgraph->n_nodes <= 1 && cgraph->n_leafs == 0) { + return false; } - const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); - const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); - if (GgmlOvDecoder::is_inp_pos(ggml_tensor, op) && GgmlOvDecoder::get_inp_pos_n_planes(op) > 1) { - // IMROPE: inp_pos stacks n_planes (t/h/w/e) position planes, each of length - // input_len; pad every plane independently so they stay aligned to chunk_size. - const int n_planes = GgmlOvDecoder::get_inp_pos_n_planes(op); - const size_t element_size = ggml_type_size(ggml_tensor->type); - ov::Shape input_shape = {1, 1, 1, (size_t) n_planes * chunk_size}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - for (int p = 0; p < n_planes; p++) { - const char * src = - (const char *) ggml_tensor->data + (p * input_len + chunk_index * chunk_size) * element_size; - char * dst = (char *) input_tensor.data() + p * chunk_size * element_size; - std::memcpy(dst, src, chunk_valid_size * element_size); - if (chunk_pad_size > 0) { - if (ggml_tensor->type == GGML_TYPE_I32) { - int32_t last_value = *((const int32_t *) src + chunk_valid_size - 1); - int32_t * out = (int32_t *) dst; - std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1); - } else if (ggml_tensor->type == GGML_TYPE_I64) { - int64_t last_value = *((const int64_t *) src + chunk_valid_size - 1); - int64_t * out = (int64_t *) dst; - std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1); - } else { - throw std::runtime_error("Unexpected tensor type for " + param_name); + // check the nodes of the model are used by the following nodes, through compare the node's use count and the count of nodes that use it as input. If does not match, return true, else return false. + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + int use_count = cgraph->use_counts[ggml_hash_find(&cgraph->visited_hash_set, node)]; + // TODO: this is a workround for the tests case from llama.cpp, fix should from the root cause in the future. + if ((cgraph->n_nodes <= 1 && use_count == 0) || + (cgraph->n_nodes <= 1 && node->op == GGML_OP_VIEW && use_count == 1 && node->src[0] != nullptr && + node->src[0]->op == GGML_OP_NONE)) { + return false; + } + if (cgraph->n_nodes == 1 && + (cgraph->nodes[0]->op == GGML_OP_TRANSPOSE || cgraph->nodes[0]->op == GGML_OP_PERMUTE)) { + return false; + } + int input_use_count = 0; + for (int j = 0; j < cgraph->n_nodes; j++) { + ggml_tensor * other_node = cgraph->nodes[j]; + for (int k = 0; k < GGML_MAX_SRC; k++) { + if (other_node->src[k] == node) { + input_use_count++; } } } - return input_tensor; + if (use_count != input_use_count && node->op != GGML_OP_NONE) { + return true; + } } - - if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || - GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { - ov::Shape input_shape = {1, 1, 1, chunk_size}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - // copy the chunk_index-th chunk from ggml_tensor - size_t element_size = ggml_type_size(ggml_tensor->type); - void * input_data = (char *) ggml_tensor->data + chunk_index * chunk_size * element_size; - std::memcpy(input_tensor.data(), input_data, chunk_valid_size * element_size); - // pad the rest with last_value + 1, so that kv's of padded positions are inserted - // to the next row after the valids row in the kvcache - if (chunk_pad_size > 0) { - if (ggml_tensor->type == GGML_TYPE_I32) { - int32_t last_value = - *((int32_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); - int32_t * output_data = input_tensor.data<int32_t>(); - std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); - } else if (ggml_tensor->type == GGML_TYPE_I64) { - int64_t last_value = - *((int64_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); - int64_t * output_data = input_tensor.data<int64_t>(); - std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); - } else { - throw std::runtime_error("Unexpected tensor type for " + param_name); - } + // if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check. + // Only weight-name membership is needed below. With GGML_OPENVINO_REDUCE_COMPILE_MEM + // use the name-only collector (no weight extraction); otherwise keep the original + // behavior of building (naive) weight nodes and take their names. + std::set<std::string> model_weights; + if (ggml_openvino_reduce_compile_mem_enabled()) { + model_weights = GgmlOvDecoder::collect_weight_names(cgraph); + } else { + for (const auto & kv : GgmlOvDecoder::create_weight_nodes(cgraph, true)) { + model_weights.insert(kv.first); } - return input_tensor; } - - if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { - size_t output_len = ggml_decoder->get_compute_params().output_len; - ov::Shape input_shape = {1, 1, 1, output_len}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - if (ggml_tensor->ne[0] == 0) { - *input_tensor.data<int32_t>() = 0; - } else { - auto * data_addr = input_tensor.data<int32_t>(); - for (size_t i = 0; i < output_len; i++) { - data_addr[i] = ((int32_t *) ggml_tensor->data)[i] % chunk_size; + std::set<ggml_tensor *> model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes); + // leaf nodes + std::set<ggml_tensor *> model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs); + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + for (int j = 0; j < GGML_MAX_SRC; j++) { + ggml_tensor * src = node->src[j]; + // the src is also not the model weights, we think the model is splitted. + // the src is also not in model leafs, we think the model is splitted. + if (src != nullptr && model_nodes.find(src) == model_nodes.end() && + model_weights.find(std::string(src->name)) == model_weights.end() && !model_leafs.empty() == false && + model_leafs.find(src) == model_leafs.end()) { + if (GgmlOvDecoder::is_inp_tok(src, node)) { + return false; + } + return true; } } - return input_tensor; } + return false; +} - if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { - size_t cols = ggml_tensor->ne[0]; - size_t rows = ggml_tensor->ne[1]; - size_t chunk_valid_rows = std::min(chunk_size, rows - chunk_index * chunk_size); - size_t context_size = ggml_decoder->get_ctx_size(); - if (ggml_tensor->type == GGML_TYPE_F16) { - const auto * ggml_data = - static_cast<const ggml_fp16_t *>(ggml_tensor->data) + chunk_index * chunk_size * cols; - std::vector<ggml_fp16_t> padded_data = pad_input<ggml_fp16_t>(ggml_data, chunk_valid_rows, cols, chunk_size, - context_size, GGML_FP32_TO_FP16(-INFINITY)); - set_zero_diagonal(padded_data, chunk_size, context_size, GGML_FP32_TO_FP16(0.0f)); - ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, chunk_size, context_size}); - std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); - return input_tensor; +bool is_naive(ggml_cgraph * cgraph) { + constexpr int naive_graph_size_threshold = 20; + int count = 0; + for (int i = 0; i < cgraph->n_nodes; i++) { + if (cgraph->nodes[i]->op != GGML_OP_NONE) { + count++; } - - const auto * ggml_data = static_cast<const float *>(ggml_tensor->data) + chunk_index * chunk_size * cols; - std::vector<float> padded_data = - pad_input<float>(ggml_data, chunk_valid_rows, cols, chunk_size, context_size, -INFINITY); - set_zero_diagonal(padded_data, chunk_size, context_size); - ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, chunk_size, context_size}); - auto * data_ptr = input_tensor.data<float>(); - std::copy(padded_data.begin(), padded_data.begin() + chunk_size * context_size, data_ptr); - return input_tensor; } - - return get_ov_input_tensor(ggml_decoder, param_name); + return count < naive_graph_size_threshold; } size_t checksum(const void * data, size_t size) { @@ -1362,15 +1645,15 @@ bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor) { std::cout << "Input name: " << name << ", Input shape: " << tensor.get_shape() << ", Address: " << tensor.data() - << std::endl; + << '\n'; switch (tensor.get_element_type()) { case ov::element::f32: { if (name.find("self_kq_mask") == std::string::npos && name.find("KQ_mask") == std::string::npos) { - std::cout << *(tensor.data<float>()) << std::endl; + std::cout << *(tensor.data<float>()) << '\n'; } else { size_t rows = tensor.get_shape()[2]; size_t cols = tensor.get_shape()[3]; - auto * data = tensor.data<float>(); + const float * data = tensor.data<float>(); for (size_t i = 0; i < rows; ++i) { for (size_t j = 0; j < cols; ++j) { float val = data[i * cols + j]; @@ -1380,26 +1663,26 @@ void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor std::cout << std::setw(5) << val; } } - std::cout << std::endl; + std::cout << '\n'; } } break; } case ov::element::f16: - std::cout << *(tensor.data<ov::float16>()) << std::endl; + std::cout << *(tensor.data<ov::float16>()) << '\n'; break; case ov::element::i32: for (size_t i = 0; i < tensor.get_size(); ++i) { - std::cout << tensor.data<int32_t>()[i] << " "; + std::cout << tensor.data<int32_t>()[i] << ' '; } - std::cout << std::endl; + std::cout << '\n'; break; case ov::element::i64: for (size_t i = 0; i < tensor.get_size(); ++i) { - std::cout << tensor.data<int64_t>()[i] << " "; + std::cout << tensor.data<int64_t>()[i] << ' '; } - std::cout << std::endl; + std::cout << '\n'; break; default: break; @@ -1408,7 +1691,7 @@ void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor void print_output_tensor_info(const std::string & name, const ov::Tensor & tensor, const void * output_dst) { std::cout << "Output name: " << name << ", Output shape: " << tensor.get_shape() << ", Address: " << output_dst - << std::endl; + << '\n'; auto print_float_stats = [](const std::string & type_name, size_t size, auto get_value) { if (size == 0) { @@ -1422,20 +1705,16 @@ void print_output_tensor_info(const std::string & name, const ov::Tensor & tenso for (size_t i = 1; i < size; ++i) { float v = get_value(i); - if (v < min) { - min = v; - } - if (v > max) { - max = v; - } + min = std::min(v, min); + max = std::max(v, max); sum += v; } double mean = sum / size; std::cout << std::right << std::setw(6) << type_name << std::right << std::setw(12) << "First" << std::setw(12) - << "Min" << std::setw(12) << "Max" << std::setw(12) << "Mean" << std::endl; + << "Min" << std::setw(12) << "Max" << std::setw(12) << "Mean" << '\n'; std::cout << std::right << std::setw(6) << "" << std::right << std::setw(12) << first << std::setw(12) << min - << std::setw(12) << max << std::setw(12) << mean << std::endl; + << std::setw(12) << max << std::setw(12) << mean << '\n'; }; switch (tensor.get_element_type()) { @@ -1492,5 +1771,3 @@ int64_t get_inp_pos_n_tokens(ggml_cgraph * cgraph, const ggml_tensor * inp_pos) bool get_is_prefill(ggml_cgraph * cgraph, const ggml_tensor * inp_pos) { return get_inp_pos_n_tokens(cgraph, inp_pos) > 1; } - -#pragma GCC diagnostic pop diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h index 5aa74da38d3b..74c25f0acea8 100644 --- a/ggml/src/ggml-openvino/utils.h +++ b/ggml/src/ggml-openvino/utils.h @@ -2,7 +2,6 @@ #include "ggml-impl.h" #include <algorithm> -#include <atomic> #include <cstddef> #include <functional> #include <memory> @@ -14,6 +13,8 @@ #include <utility> #include <vector> +// Local execution-cache key. A match still needs the ModelParams compatibility +// check; this key alone does not identify weights or a compiled model. struct graph_key { int n_nodes; std::string first_node_name; @@ -26,14 +27,13 @@ struct graph_key { last_node_name = cgraph->nodes[n_nodes - 1]->name; } - auto get_input_key_name = [](const ggml_cgraph * graph, const ggml_tensor * tensor) { - std::string name = tensor->name; - const size_t hash_pos = ggml_hash_find(&graph->visited_hash_set, tensor); - if (((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || GgmlOvDecoder::is_kvcache(tensor, nullptr)) && - hash_pos != GGML_HASHSET_FULL && ggml_bitset_get(graph->visited_hash_set.used, hash_pos)) { - name += "#" + std::to_string(hash_pos); + std::unordered_map<const ggml_tensor *, std::string> names; + auto get_input_key_name = [&names](const ggml_cgraph * graph, const ggml_tensor * tensor) { + auto it = names.find(tensor); + if (it == names.end()) { + it = names.emplace(tensor, GgmlOvDecoder::get_tensor_name(graph, tensor)).first; } - return name; + return it->second; }; std::vector<std::string> node_names; @@ -90,7 +90,27 @@ struct decoder_runtime_ctx { std::shared_ptr<GgmlOvDecoder> ptr; }; +struct ov_compiled_graph { + ov::CompiledModel decode; + ov::CompiledModel prefill; + std::vector<std::string> input_names; + std::vector<std::string> output_names; +}; + +// Only compilation and cache publication use this mutex. Requests, decoders and +// sequence state belong to individual backend contexts and never enter this cache. +struct ov_compiled_model_cache { + std::mutex mutex; + std::unordered_map<std::string, ov_compiled_graph> graphs; + size_t backend_count = 0; +}; + +// Private to one backend instance. Only compiled_cache is shared with other +// instances; clearing these local caches cannot invalidate their requests. struct ov_runtime_context { + // Serializes calls on this backend only, not inference in other contexts. + std::mutex execution_mutex; + std::shared_ptr<ov_compiled_model_cache> compiled_cache; mutable std::mutex ctx_mutex; std::string device; bool stateful; @@ -99,13 +119,10 @@ struct ov_runtime_context { std::unordered_map<graph_key, std::shared_ptr<ov::InferRequest>, graph_key_hash> infer_request_cache_prefill; std::unordered_map<graph_key, std::vector<std::string>, graph_key_hash> ov_input_names_cache; std::unordered_map<graph_key, std::vector<std::string>, graph_key_hash> ov_output_names_cache; - //TODO: Stateful is only supported for single request at a time. - // Simultanous stateful inference request support to be added. size_t stateful_kv_size; std::map<std::string, std::string> kv_state_input_name_map; - std::atomic<int> backend_count; - ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0), backend_count(0) {} + ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0) {} void clear_caches_locked() { decoder_cache.clear(); @@ -125,9 +142,6 @@ struct ov_runtime_context { enum ggml_status ov_graph_compute(struct ggml_cgraph * cgraph, ggml_backend_t backend); -enum ggml_status ov_graph_compute_dynamic(struct ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx); -enum ggml_status ov_graph_compute_static(struct ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx); - size_t checksum(const void * data, size_t size); bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string & file_path); @@ -168,18 +182,6 @@ int64_t get_inp_pos_n_tokens(struct ggml_cgraph * cgraph, const ggml_tensor * in bool get_is_prefill(struct ggml_cgraph * cgraph, const ggml_tensor * inp_pos); -ov::Tensor get_ov_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & param_name); -ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr<GgmlOvDecoder> ggml_decoder, - const std::string & param_name); -ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr<GgmlOvDecoder> ggml_decoder, - const std::string & param_name, - int chunk_index); - -ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, - std::shared_ptr<ov::InferRequest> infer_request, - int output_index, - const ggml_tensor * ggml_tensor); - bool is_naive(struct ggml_cgraph * cgraph); /** @@ -188,8 +190,3 @@ bool is_naive(struct ggml_cgraph * cgraph); * @return true if the graph is identified as split; otherwise false. */ bool is_model_splitted(struct ggml_cgraph * cgraph); - -enum ggml_status naive_compute(struct ggml_cgraph * cgraph, - ov::Core & core, - const std::string & device, - const ov::AnyMap & config); diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index df905bc94261..32f9165080f8 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -4771,80 +4771,51 @@ static void quantize_row_iq1_m_impl(const float * GGML_RESTRICT x, void * GGML_R // 1: +, - // 2: -, + // 3: -, - - for (int i1 = 0; i1 <= block_size; ++i1) { - for (int i2 = i1; i2 <= block_size; ++i2) { - memset(sumqx, 0, 4*sizeof(float)); - memset(sumq2, 0, 4*sizeof(float)); - for (int j = 0; j < i1; ++j) { - int i = idx[2*j]; - if (i < block_size/2) { - sumqx[0] += weight[i]*x_p[0]*xb[i]; - sumqx[1] += weight[i]*x_p[0]*xb[i]; - sumqx[2] += weight[i]*x_m[0]*xb[i]; - sumqx[3] += weight[i]*x_m[0]*xb[i]; - sumq2[0] += weight[i]*x_p[0]*x_p[0]; - sumq2[1] += weight[i]*x_p[0]*x_p[0]; - sumq2[2] += weight[i]*x_m[0]*x_m[0]; - sumq2[3] += weight[i]*x_m[0]*x_m[0]; - } else { - sumqx[0] += weight[i]*x_p[0]*xb[i]; - sumqx[2] += weight[i]*x_p[0]*xb[i]; - sumqx[1] += weight[i]*x_m[0]*xb[i]; - sumqx[3] += weight[i]*x_m[0]*xb[i]; - sumq2[0] += weight[i]*x_p[0]*x_p[0]; - sumq2[2] += weight[i]*x_p[0]*x_p[0]; - sumq2[1] += weight[i]*x_m[0]*x_m[0]; - sumq2[3] += weight[i]*x_m[0]*x_m[0]; - } - } - for (int j = i1; j < i2; ++j) { - int i = idx[2*j]; - if (i < block_size/2) { - sumqx[0] += weight[i]*x_p[1]*xb[i]; - sumqx[1] += weight[i]*x_p[1]*xb[i]; - sumqx[2] += weight[i]*x_m[1]*xb[i]; - sumqx[3] += weight[i]*x_m[1]*xb[i]; - sumq2[0] += weight[i]*x_p[1]*x_p[1]; - sumq2[1] += weight[i]*x_p[1]*x_p[1]; - sumq2[2] += weight[i]*x_m[1]*x_m[1]; - sumq2[3] += weight[i]*x_m[1]*x_m[1]; - } else { - sumqx[0] += weight[i]*x_p[1]*xb[i]; - sumqx[2] += weight[i]*x_p[1]*xb[i]; - sumqx[1] += weight[i]*x_m[1]*xb[i]; - sumqx[3] += weight[i]*x_m[1]*xb[i]; - sumq2[0] += weight[i]*x_p[1]*x_p[1]; - sumq2[2] += weight[i]*x_p[1]*x_p[1]; - sumq2[1] += weight[i]*x_m[1]*x_m[1]; - sumq2[3] += weight[i]*x_m[1]*x_m[1]; - } - } - for (int j = i2; j < block_size; ++j) { - int i = idx[2*j]; - if (i < block_size/2) { - sumqx[0] += weight[i]*x_p[2]*xb[i]; - sumqx[1] += weight[i]*x_p[2]*xb[i]; - sumqx[2] += weight[i]*x_m[2]*xb[i]; - sumqx[3] += weight[i]*x_m[2]*xb[i]; - sumq2[0] += weight[i]*x_p[2]*x_p[2]; - sumq2[1] += weight[i]*x_p[2]*x_p[2]; - sumq2[2] += weight[i]*x_m[2]*x_m[2]; - sumq2[3] += weight[i]*x_m[2]*x_m[2]; - } else { - sumqx[0] += weight[i]*x_p[2]*xb[i]; - sumqx[2] += weight[i]*x_p[2]*xb[i]; - sumqx[1] += weight[i]*x_m[2]*xb[i]; - sumqx[3] += weight[i]*x_m[2]*xb[i]; - sumq2[0] += weight[i]*x_p[2]*x_p[2]; - sumq2[2] += weight[i]*x_p[2]*x_p[2]; - sumq2[1] += weight[i]*x_m[2]*x_m[2]; - sumq2[3] += weight[i]*x_m[2]*x_m[2]; + // prefix sums are kept per half of the block because each half can use a different sign (x_p or x_m) + // since v[0]-v[1] = v[1]-v[2] = -1 for both x_p and x_m, the 3-group sum for a split collapses to T*v[2] - px[i1] - px[i2] + { + float px[2][IQ1M_BLOCK_SIZE+1]; + float pw[2][IQ1M_BLOCK_SIZE+1]; + px[0][0] = px[1][0] = 0; + pw[0][0] = pw[1][0] = 0; + for (int j = 0; j < block_size; ++j) { + const int i = idx[2*j]; + const int h = i < block_size/2 ? 0 : 1; + px[h][j+1] = px[h][j] + weight[i]*xb[i]; + px[1-h][j+1] = px[1-h][j]; + pw[h][j+1] = pw[h][j] + weight[i]; + pw[1-h][j+1] = pw[1-h][j]; + } + const float txs[2] = {px[0][block_size], px[1][block_size]}; // total weight*x per half + const float tws[2] = {pw[0][block_size], pw[1][block_size]}; // total weight per half + const float p2 = x_p[2], m2 = x_m[2]; + const float cp1 = x_p[0]*x_p[0] - x_p[1]*x_p[1]; + const float cp2 = x_p[1]*x_p[1] - x_p[2]*x_p[2]; + const float cm1 = x_m[0]*x_m[0] - x_m[1]*x_m[1]; + const float cm2 = x_m[1]*x_m[1] - x_m[2]*x_m[2]; + for (int i1 = 0; i1 <= block_size; ++i1) { + for (int i2 = i1; i2 <= block_size; ++i2) { + float qx_p[2], qx_m[2], q2_p[2], q2_m[2]; + for (int h = 0; h < 2; ++h) { + const float sx = px[h][i1] + px[h][i2]; + qx_p[h] = txs[h]*p2 - sx; + qx_m[h] = txs[h]*m2 - sx; + q2_p[h] = tws[h]*p2*p2 + pw[h][i1]*cp1 + pw[h][i2]*cp2; + q2_m[h] = tws[h]*m2*m2 + pw[h][i1]*cm1 + pw[h][i2]*cm2; } - } - for (int k = 0; k < 4; ++k) { - if (sumq2[k] > 0 && sumqx[k]*sumqx[k] > best_score*sumq2[k]) { - scale = sumqx[k]/sumq2[k]; best_score = scale*sumqx[k]; - besti1 = i1; besti2 = i2; best_k = k; + sumqx[0] = qx_p[0] + qx_p[1]; + sumqx[1] = qx_p[0] + qx_m[1]; + sumqx[2] = qx_m[0] + qx_p[1]; + sumqx[3] = qx_m[0] + qx_m[1]; + sumq2[0] = q2_p[0] + q2_p[1]; + sumq2[1] = q2_p[0] + q2_m[1]; + sumq2[2] = q2_m[0] + q2_p[1]; + sumq2[3] = q2_m[0] + q2_m[1]; + for (int k = 0; k < 4; ++k) { + if (sumq2[k] > 0 && sumqx[k]*sumqx[k] > best_score*sumq2[k]) { + scale = sumqx[k]/sumq2[k]; best_score = scale*sumqx[k]; + besti1 = i1; besti2 = i2; best_k = k; + } } } } diff --git a/ggml/src/ggml-rpc/CMakeLists.txt b/ggml/src/ggml-rpc/CMakeLists.txt index af3bd0290f7c..e3d0c9b4c200 100644 --- a/ggml/src/ggml-rpc/CMakeLists.txt +++ b/ggml/src/ggml-rpc/CMakeLists.txt @@ -36,8 +36,10 @@ if (GGML_RPC_RDMA) target_compile_definitions(ggml-rpc PRIVATE GGML_RPC_RDMA) if (APPLE) # librdma.dylib only exists on macOS 26.2 and later. Link it weakly so a build made - # where it exists still loads where it does not; checked at runtime before use. - target_link_options(ggml-rpc PRIVATE "LINKER:-weak_library,${RDMA_LIB}") + # where it exists still loads where it does not; checked at runtime before use + # but with BUILD_SHARED_LIBS=OFF ggml-rpc is a static archive and never links + # so the librdma symbols used by transport-apple.cpp stay undefined. + target_link_options(ggml-rpc PUBLIC "LINKER:-weak_library,${RDMA_LIB}") target_compile_definitions(ggml-rpc PRIVATE GGML_RPC_RDMA_APPLE) target_sources(ggml-rpc PRIVATE transport-apple.cpp) else() diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index cc7d7206933f..c24caad77ee9 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -697,10 +697,31 @@ static void ggml_backend_rpc_buffer_memset_tensor( ctx->dispatcher->send(RPC_CMD_MEMSET_TENSOR, request, sizeof(*request)); } +// input serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) +static std::shared_ptr<uint8_t> serialize_set_tensor(const rpc_tensor & rpc_tensor, uint8_t cache_flag, uint64_t offset, const void * data, size_t size, size_t & input_size) { + input_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset) + size; + uint8_t * input = new uint8_t[input_size](); + uint8_t * p = input; + memcpy(p, &rpc_tensor, sizeof(rpc_tensor)); p += sizeof(rpc_tensor); + memcpy(p, &cache_flag, sizeof(cache_flag)); p += sizeof(cache_flag); + memcpy(p, &offset, sizeof(offset)); p += sizeof(offset); + memcpy(p, data, size); + return std::shared_ptr<uint8_t>(input, std::default_delete<uint8_t[]>()); +} + +// the hash cache is meant for weights, so that a model reload can skip re-sending them. +// compute-buffer inputs (the activations ggml_backend_sched copies between backends) must not +// take this path, otherwise with `rpc-server -c` every ubatch above the threshold is written +// to the cache directory and later served from there. +static bool rpc_use_hash_cache(const ggml_tensor * tensor, size_t size) { + return size > HASH_THRESHOLD && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; +} + static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; rpc_tensor rpc_tensor = serialize_tensor(tensor); - if (size > HASH_THRESHOLD) { + uint8_t cache_flag = 0; + if (rpc_use_hash_cache(tensor, size)) { auto request = std::make_shared<rpc_msg_set_tensor_hash_req>(); request->tensor = rpc_tensor; request->offset = offset; @@ -711,15 +732,12 @@ static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggm // the server has the same data, no need to send it return; } + // the server has no cache entry for this tensor - ask it to save one + cache_flag = 1; } - // input serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) - size_t input_size = sizeof(rpc_tensor) + sizeof(uint64_t) + size; - uint8_t * input = new uint8_t[input_size](); - memcpy(input, &rpc_tensor, sizeof(rpc_tensor)); - memcpy(input + sizeof(rpc_tensor), &offset, sizeof(offset)); - memcpy(input + sizeof(rpc_tensor) + sizeof(offset), data, size); - std::shared_ptr<uint8_t> input_ptr(input, std::default_delete<uint8_t[]>()); - ctx->dispatcher->send(RPC_CMD_SET_TENSOR, input_ptr, input_size); + size_t input_size; + auto input = serialize_set_tensor(rpc_tensor, cache_flag, offset, data, size, input_size); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR, input, input_size); } static void ggml_backend_rpc_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { @@ -927,7 +945,8 @@ static void ggml_backend_rpc_free(ggml_backend_t backend) { static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_rpc_context * ctx = (ggml_backend_rpc_context *)backend->context; rpc_tensor rpc_tensor = serialize_tensor(tensor); - if (size > HASH_THRESHOLD) { + uint8_t cache_flag = 0; + if (rpc_use_hash_cache(tensor, size)) { auto request = std::make_shared<rpc_msg_set_tensor_hash_req>(); request->tensor = rpc_tensor; request->offset = offset; @@ -939,15 +958,12 @@ static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tenso // the server has the same data, no need to send it return; } + // the server has no cache entry for this tensor - ask it to save one + cache_flag = 1; } - // input serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) - size_t input_size = sizeof(rpc_tensor) + sizeof(uint64_t) + size; - uint8_t * input = new uint8_t[input_size](); - memcpy(input, &rpc_tensor, sizeof(rpc_tensor)); - memcpy(input + sizeof(rpc_tensor), &offset, sizeof(offset)); - memcpy(input + sizeof(rpc_tensor) + sizeof(offset), data, size); - std::shared_ptr<uint8_t> input_ptr(input, std::default_delete<uint8_t[]>()); - ctx->dispatcher->send_async(RPC_CMD_SET_TENSOR, input_ptr, input_size); + size_t input_size; + auto input = serialize_set_tensor(rpc_tensor, cache_flag, offset, data, size, input_size); + ctx->dispatcher->send_async(RPC_CMD_SET_TENSOR, input, input_size); } static void ggml_backend_rpc_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { @@ -1285,6 +1301,11 @@ bool rpc_server::free_buffer(const rpc_msg_free_buffer_req & request) { GGML_LOG_ERROR("[%s] buffer not found\n", __func__); return false; } + // Discard all cached graphs to avoid use-after-free in graph_recompute, + // since their nodes may hold pointers to the buffer being freed. + for (auto & sg : stored_graphs) { + sg.graph = nullptr; + } ggml_backend_buffer_free(buffer); buffers.erase(buffer); return true; @@ -1398,14 +1419,17 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp bool rpc_server::set_tensor(const std::vector<uint8_t> & input) { - // serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) | - if (input.size() < sizeof(rpc_tensor) + sizeof(uint64_t)) { + // serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) | + uint8_t cache_flag; + uint64_t offset; + const size_t header_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset); + if (input.size() < header_size) { return false; } const rpc_tensor * in_tensor = (const rpc_tensor *)input.data(); - uint64_t offset; - memcpy(&offset, input.data() + sizeof(rpc_tensor), sizeof(offset)); - const size_t size = input.size() - sizeof(rpc_tensor) - sizeof(offset); + memcpy(&cache_flag, input.data() + sizeof(rpc_tensor), sizeof(cache_flag)); + memcpy(&offset, input.data() + sizeof(rpc_tensor) + sizeof(cache_flag), sizeof(offset)); + const size_t size = input.size() - header_size; struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), @@ -1434,8 +1458,8 @@ bool rpc_server::set_tensor(const std::vector<uint8_t> & input) { } } - const void * data = input.data() + sizeof(rpc_tensor) + sizeof(offset); - if (cache_dir && size > HASH_THRESHOLD) { + const void * data = input.data() + header_size; + if (cache_dir && cache_flag) { uint64_t hash = fnv_hash((const uint8_t*)data, size); char hash_str[17]; snprintf(hash_str, sizeof(hash_str), "%016" PRIx64, hash); @@ -1733,7 +1757,6 @@ bool rpc_server::graph_compute(const std::vector<uint8_t> & input) { int64_t id; memcpy(&id, &nodes[i], sizeof(id)); graph->nodes[i] = create_node(id, ctx, tensor_ptrs, tensor_map); - // Check if create_node failed for a *non-zero* ID. // If id was 0, create_node returning nullptr is expected. // If id was non-zero and create_node returned nullptr, it indicates a deserialization error. diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index a8d9c0d804bf..d2196f74d565 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -110,15 +110,21 @@ if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API) # Link against Level Zero loader for direct device memory allocation. # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging # in the xe kernel driver during multi-GPU inference. - find_path(LEVEL_ZERO_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include) + find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include) find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH) - if(ZE_LOADER_LIB AND LEVEL_ZERO_INCLUDE_DIR) + if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR) target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB}) target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API) message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}") - message(STATUS "Level Zero headers found: ${LEVEL_ZERO_INCLUDE_DIR}") + message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}") else() - message(WARNING "Level Zero loader or headers not found, Level Zero support disabled") + message(WARNING "Level Zero loader or development headers not found, " + "Level Zero API support disabled. " + "Please install the Level Zero SDK/development package " + "to support Level Zero API features. " + "Level Zero API is not mandatory for SYCL backend, " + "but it is required by the special features for better " + "function & performance on Intel GPUs.") endif() endif() diff --git a/ggml/src/ggml-sycl/backend.hpp b/ggml/src/ggml-sycl/backend.hpp index 51ab6f930dca..ab80a2a3b941 100644 --- a/ggml/src/ggml-sycl/backend.hpp +++ b/ggml/src/ggml-sycl/backend.hpp @@ -44,6 +44,7 @@ #include "ssm_conv.hpp" #include "softmax.hpp" #include "topk-moe.hpp" +#include "topk-radix.hpp" #include "tsembd.hpp" #include "upscale.hpp" #include "wkv.hpp" diff --git a/ggml/src/ggml-sycl/base.hpp b/ggml/src/ggml-sycl/base.hpp index 3afd57ccb2db..fe96c4ab855e 100644 --- a/ggml/src/ggml-sycl/base.hpp +++ b/ggml/src/ggml-sycl/base.hpp @@ -17,6 +17,7 @@ #include <cstdio> extern int g_ggml_sycl_debug; +extern int g_ggml_sycl_dev_debug; #if defined(__clang__) && __has_builtin(__builtin_expect) // Hint the optimizer to pipeline the more likely following instruction in branches @@ -33,4 +34,10 @@ extern int g_ggml_sycl_debug; fprintf(stderr, __VA_ARGS__); \ } while (0) +#define GGML_SYCL_DEV_DEBUG(...) \ + do { \ + if (UNLIKELY(g_ggml_sycl_dev_debug)) \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) + #endif // GGML_SYCL_BASE_HPP diff --git a/ggml/src/ggml-sycl/common.cpp b/ggml/src/ggml-sycl/common.cpp index e1b6db13eb41..894006949d23 100644 --- a/ggml/src/ggml-sycl/common.cpp +++ b/ggml/src/ggml-sycl/common.cpp @@ -94,7 +94,7 @@ static bool ggml_sycl_use_level_zero_device_alloc(sycl::queue &q) { // Use Level Zero zeMemAllocDevice to avoid sycl::malloc_device triggering // DMA-buf/TTM system RAM staging in the xe kernel driver during multi-GPU inference. -void * ggml_sycl_malloc_device(size_t size, sycl::queue &q) { +void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type type) { #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API if (ggml_sycl_use_level_zero_device_alloc(q)) { void *ptr = nullptr; @@ -117,16 +117,25 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q) { #endif ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, 64, ze_dev, &ptr); if (r == ZE_RESULT_SUCCESS && ptr) { + ggml_sycl_memtrace_add(type, ptr, size); return ptr; } + ggml_sycl_memtrace_fail(type, size); return nullptr; } #endif - return sycl::malloc_device(size, q); + void * ptr = sycl::malloc_device(size, q); + if (ptr == nullptr) { + ggml_sycl_memtrace_fail(type, size); + return nullptr; + } + ggml_sycl_memtrace_add(type, ptr, size); + return ptr; } void ggml_sycl_free_device(void *ptr, sycl::queue &q) { if (!ptr) return; + ggml_sycl_memtrace_del(ptr); #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API if (ggml_sycl_use_level_zero_device_alloc(q)) { auto ze_ctx = sycl::get_native<sycl::backend::ext_oneapi_level_zero>(q.get_context()); diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index 9f2a27b18e06..dc6cdd3df462 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -27,6 +27,7 @@ #include "type.hpp" #include "sycl_hw.hpp" #include "fattn-buffers.hpp" +#include "memtrace.hpp" namespace syclexp = sycl::ext::oneapi::experimental; @@ -69,6 +70,8 @@ extern int g_ggml_sycl_dev2dev_memcpy; extern int g_ggml_sycl_fa_onednn; extern int g_ggml_sycl_fa_onednn_max_kv; extern int g_ggml_sycl_enable_mkl_fa; +extern int g_ggml_sycl_memtrace; +extern int g_ggml_sycl_memtrace_step; #define CHECK_TRY_ERROR(expr) \ @@ -318,7 +321,8 @@ struct ggml_tensor_extra_gpu { }; extern int g_ggml_sycl_use_level_zero_api; -void * ggml_sycl_malloc_device(size_t size, sycl::queue &q); +void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, + ggml_sycl_mem_type type = GGML_SYCL_MEM_DIRECT); void ggml_sycl_free_device(void *ptr, sycl::queue &q); void release_extra_gpu(ggml_tensor_extra_gpu * extra, std::vector<queue_ptr> streams={}); @@ -397,29 +401,10 @@ struct ggml_backend_sycl_context { dnnl::stream stream_dnnl() { return stream_dnnl(device, 0); } - dnnl::memory get_scratchpad_mem(const dnnl::memory::desc & scratchpad_md, - const dnnl::engine & eng, const queue_ptr q) { - ggml_sycl_pool_alloc<uint8_t> * pool; - auto it = scratchpad_map.find(q); - if (it == scratchpad_map.end()) { - scratchpad_map[q] = std::make_unique<ggml_sycl_pool_alloc<uint8_t>>(this->pool()); - pool = scratchpad_map[q].get(); - } else { - pool = it->second.get(); - } - - size_t scratchpad_size = scratchpad_md.get_size(); - if (scratchpad_size > pool->actual_size) { - pool->realloc(scratchpad_size); - } - void * mem_ptr = pool->get(); - return dnnl::memory(scratchpad_md, eng, mem_ptr); - } #endif // pool std::unique_ptr<ggml_sycl_pool> pools[GGML_SYCL_MAX_DEVICES]; - std::unordered_map<sycl::queue *, std::unique_ptr<ggml_sycl_pool_alloc<uint8_t>>> scratchpad_map; std::unique_ptr<ggml_sycl_fattn_kv_buffers> fattn_bufs[GGML_SYCL_MAX_DEVICES]; diff --git a/ggml/src/ggml-sycl/dsv4-hc.cpp b/ggml/src/ggml-sycl/dsv4-hc.cpp index bb66e8c1b43b..337f4af4559e 100644 --- a/ggml/src/ggml-sycl/dsv4-hc.cpp +++ b/ggml/src/ggml-sycl/dsv4-hc.cpp @@ -2,22 +2,30 @@ #include "dsv4-hc.hpp" #include <cmath> +#include <type_traits> static constexpr int DSV4_HC = 4; +// tunable: one work-item per (embedding element, token) +static constexpr int dsv4_hc_pre_block_size = 256; + +// gated: the weight is a per-element gate [n_embd, hc, n_tokens] passed through a sigmoid. +// otherwise it is one weight per (stream, token). +template <bool gated> static void dsv4_hc_pre_f32_sycl( const float * x, const float * weights, float * dst, int64_t n_embd, int64_t hc, int64_t n_tokens, int64_t sx0, int64_t sx1, int64_t sx2, - int64_t sw0, int64_t sw1, + int64_t sw0, int64_t sw1, int64_t sw2, int64_t sd0, int64_t sd1, + float scale, queue_ptr stream) { const int64_t nr = n_embd * n_tokens; - const int64_t block_size = 256; - const int64_t num_blocks = (nr + block_size - 1) / block_size; + const int64_t num_blocks = (nr + dsv4_hc_pre_block_size - 1) / dsv4_hc_pre_block_size; stream->parallel_for( - sycl::nd_range<1>(sycl::range<1>(num_blocks * block_size), sycl::range<1>(block_size)), + sycl::nd_range<1>(sycl::range<1>(num_blocks * dsv4_hc_pre_block_size), + sycl::range<1>(dsv4_hc_pre_block_size)), [=](sycl::nd_item<1> item) { const int64_t ir = item.get_global_id(0); if (ir >= nr) { @@ -27,14 +35,20 @@ static void dsv4_hc_pre_f32_sycl( const int64_t i0 = ir % n_embd; const int64_t it = ir / n_embd; - float sum = x[i0*sx0 + it*sx2] * weights[it*sw1]; - for (int64_t ih = 1; ih < hc; ++ih) { + float sum = 0.0f; + for (int64_t ih = 0; ih < hc; ++ih) { const float xv = x[i0*sx0 + ih*sx1 + it*sx2]; - const float wv = weights[ih*sw0 + it*sw1]; + float wv; + if constexpr (gated) { + const float gv = weights[i0*sw0 + ih*sw1 + it*sw2]; + wv = 1.0f / (1.0f + sycl::exp(-gv)); + } else { + wv = weights[ih*sw0 + it*sw1]; + } sum += xv * wv; } - dst[i0*sd0 + it*sd1] = sum; + dst[i0*sd0 + it*sd1] = scale * sum; }); } @@ -138,6 +152,12 @@ static void dsv4_hc_comb_f32_sycl( }); } +// tunable: one work-item per (embedding element, stream, token) +static constexpr int dsv4_hc_post_block_size = 256; + +// comb == nullptr is identity mixing: each destination stream keeps its own residual +// instead of summing across the streams. +template <bool has_comb> static void dsv4_hc_post_f32_sycl( const float * x, const float * residual, const float * post, const float * comb, float * dst, int64_t n_embd, int64_t hc, int64_t n_tokens, @@ -148,7 +168,7 @@ static void dsv4_hc_post_f32_sycl( int64_t sd0, int64_t sd1, int64_t sd2, queue_ptr stream) { const int64_t nr = n_embd * hc * n_tokens; - const int64_t block_size = 256; + const int64_t block_size = dsv4_hc_post_block_size; const int64_t num_blocks = (nr + block_size - 1) / block_size; stream->parallel_for( @@ -164,8 +184,12 @@ static void dsv4_hc_post_f32_sycl( const int64_t it = ir / (n_embd * hc); float sum = x[i0*sx0 + it*sx1] * post[idst*sp0 + it*sp1]; - for (int64_t isrc = 0; isrc < hc; ++isrc) { - sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2]; + if constexpr (has_comb) { + for (int64_t isrc = 0; isrc < hc; ++isrc) { + sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2]; + } + } else { + sum += residual[i0*sr0 + idst*sr1 + it*sr2]; } dst[i0*sd0 + idst*sd1 + it*sd2] = sum; @@ -189,15 +213,33 @@ void ggml_sycl_op_dsv4_hc_pre(ggml_backend_sycl_context & ctx, ggml_tensor * dst const int64_t hc = x->ne[1]; const int64_t n_tokens = x->ne[2]; + const float scale = ggml_get_op_params_f32(dst, 0); + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + queue_ptr stream = ctx.stream(); - dsv4_hc_pre_f32_sycl( - (const float *) x->data, (const float *) weights->data, (float *) dst->data, - n_embd, hc, n_tokens, - nbx0 / sizeof(float), nbx1 / sizeof(float), nbx2 / sizeof(float), - nbw0 / sizeof(float), nbw1 / sizeof(float), - nbd0 / sizeof(float), nbd1 / sizeof(float), - stream); + if (gated) { + GGML_ASSERT(weights->ne[0] == n_embd); + GGML_ASSERT(weights->ne[1] == hc); + GGML_ASSERT(weights->ne[2] == n_tokens); + dsv4_hc_pre_f32_sycl<true>( + (const float *) x->data, (const float *) weights->data, (float *) dst->data, + n_embd, hc, n_tokens, + nbx0 / sizeof(float), nbx1 / sizeof(float), nbx2 / sizeof(float), + nbw0 / sizeof(float), nbw1 / sizeof(float), nbw2 / sizeof(float), + nbd0 / sizeof(float), nbd1 / sizeof(float), + scale, stream); + } else { + GGML_ASSERT(weights->ne[0] == hc); + GGML_ASSERT(weights->ne[1] == n_tokens); + dsv4_hc_pre_f32_sycl<false>( + (const float *) x->data, (const float *) weights->data, (float *) dst->data, + n_embd, hc, n_tokens, + nbx0 / sizeof(float), nbx1 / sizeof(float), nbx2 / sizeof(float), + nbw0 / sizeof(float), nbw1 / sizeof(float), /*sw2=*/ 0, + nbd0 / sizeof(float), nbd1 / sizeof(float), + scale, stream); + } } void ggml_sycl_op_dsv4_hc_comb(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { @@ -252,24 +294,33 @@ void ggml_sycl_op_dsv4_hc_post(ggml_backend_sycl_context & ctx, ggml_tensor * ds GGML_ASSERT(x->type == GGML_TYPE_F32); GGML_ASSERT(residual->type == GGML_TYPE_F32); GGML_ASSERT(post->type == GGML_TYPE_F32); - GGML_ASSERT(comb->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); GGML_TENSOR_LOCALS(size_t, nbx, x, nb); GGML_TENSOR_LOCALS(size_t, nbr, residual, nb); GGML_TENSOR_LOCALS(size_t, nbp, post, nb); - GGML_TENSOR_LOCALS(size_t, nbc, comb, nb); GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + size_t nbc0 = 0; + size_t nbc1 = 0; + size_t nbc2 = 0; + if (comb) { + GGML_ASSERT(comb->type == GGML_TYPE_F32); + nbc0 = comb->nb[0]; + nbc1 = comb->nb[1]; + nbc2 = comb->nb[2]; + } + const int64_t n_embd = x->ne[0]; const int64_t n_tokens = x->ne[1]; const int64_t hc = residual->ne[1]; queue_ptr stream = ctx.stream(); - dsv4_hc_post_f32_sycl( + const auto launch = [&](auto has_comb) { + dsv4_hc_post_f32_sycl<decltype(has_comb)::value>( (const float *) x->data, (const float *) residual->data, - (const float *) post->data, (const float *) comb->data, (float *) dst->data, + (const float *) post->data, comb ? (const float *) comb->data : nullptr, (float *) dst->data, n_embd, hc, n_tokens, nbx0 / sizeof(float), nbx1 / sizeof(float), nbr0 / sizeof(float), nbr1 / sizeof(float), nbr2 / sizeof(float), @@ -277,4 +328,11 @@ void ggml_sycl_op_dsv4_hc_post(ggml_backend_sycl_context & ctx, ggml_tensor * ds nbc0 / sizeof(float), nbc1 / sizeof(float), nbc2 / sizeof(float), nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float), stream); + }; + + if (comb) { + launch(std::true_type{}); + } else { + launch(std::false_type{}); + } } diff --git a/ggml/src/ggml-sycl/fattn-buffers.cpp b/ggml/src/ggml-sycl/fattn-buffers.cpp index 46cf6d551f17..78a52d2ab7f4 100644 --- a/ggml/src/ggml-sycl/fattn-buffers.cpp +++ b/ggml/src/ggml-sycl/fattn-buffers.cpp @@ -21,6 +21,7 @@ sycl::half * ggml_sycl_fattn_kv_buffers::kv_buffer::ensure_half(size_t n_elems) if (ptr) { SYCL_CHECK(CHECK_TRY_ERROR(qptr->wait())); + ggml_sycl_memtrace_del(ptr); SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr))); ptr = nullptr; capacity = 0; @@ -38,11 +39,13 @@ sycl::half * ggml_sycl_fattn_kv_buffers::kv_buffer::ensure_half(size_t n_elems) if (!dev_ptr) { GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device\n", __func__, cap); + ggml_sycl_memtrace_fail(GGML_SYCL_MEM_FATTN_KV, cap); GGML_ABORT("fattn buffer alloc failed"); } ptr = static_cast<sycl::half *>(dev_ptr); capacity = cap; + ggml_sycl_memtrace_add(GGML_SYCL_MEM_FATTN_KV, ptr, cap); return ptr; } @@ -51,6 +54,7 @@ ggml_sycl_fattn_kv_buffers::kv_buffer::~kv_buffer() { GGML_LOG_INFO("ggml_sycl_fattn_kv_buffer[%d]: %.2f MiB\n", device, capacity / 1024.0 / 1024.0); #endif if (ptr) { + ggml_sycl_memtrace_del(ptr); SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr))); } } diff --git a/ggml/src/ggml-sycl/fattn-mkl.cpp b/ggml/src/ggml-sycl/fattn-mkl.cpp index 2d164a0840fd..30947b17bf1b 100644 --- a/ggml/src/ggml-sycl/fattn-mkl.cpp +++ b/ggml/src/ggml-sycl/fattn-mkl.cpp @@ -110,8 +110,15 @@ static void mkl_fa_init_softmax_state( // The tile spans absolute rows [q0, q0 + q_rows). Score buffers // (KQ_f32/S_f16) are indexed RELATIVE to the tile; the persistent state // (VKQ_accum/KQ_max/KQ_sum) and mask are indexed by ABSOLUTE row. -// For each row: find local max → rescale previous VKQ_accum → -// compute exp(s - max) → write S_f16 → update running max/sum. +// One WORK-GROUP per query row (local size = wg_size): work-items stride +// over the chunk so adjacent items touch adjacent elements (coalesced), +// the row max/sum come from group reductions, and the DV-long VKQ +// rescale is spread across the items. Item 0 is the sole writer of +// KQ_max/KQ_sum; its writes are ordered after every other item's reads +// by the second group reduction (a collective). Per-element math is +// identical to the original one-item-per-row kernel: softcap before +// mask, native::exp, -1e30 sentinel, half-precision S. Only the float +// summation order differs (tree vs serial), i.e. last-ulp level. static void mkl_fa_online_softmax_chunk( dpct::queue_ptr stream, float * __restrict KQ_f32, @@ -126,25 +133,27 @@ static void mkl_fa_online_softmax_chunk( int64_t mask_row_stride, int mask_n_heads, float logit_softcap, int64_t wg_size) { - const int64_t wg = ((q_rows + wg_size - 1) / wg_size) * wg_size; - + // One work-group per query row: exactly q_rows groups of wg_size + // items. q_rows * wg_size is already a multiple of wg_size, so unlike + // the one-item-per-row kernels there is no round-up / tail guard. + const int64_t wg = q_rows * wg_size; + const int local_size = (int) wg_size; // stride in the loops below stream->submit([&](sycl::handler & cgh) { cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), [=](sycl::nd_item<1> item) { - int jc_rel = item.get_global_id(0); - if (jc_rel >= q_rows) return; - int jc_abs = q0 + jc_rel; - + const int local_id = (int)item.get_local_id(0); + const int row = (int)item.get_group(0); // tile-relative + const int jc_abs = q0 + row; const int gqa_group = jc_abs / n_queries; const int q_row = jc_abs % n_queries; - // Score buffers are tile-local (relative index). const float * __restrict KQ_row = KQ_f32 - + jc_rel * (int64_t)chunk_size; + + row * (int64_t)chunk_size; + sycl::half * __restrict S_row = S_f16 + + row * (int64_t)chunk_size; // Persistent accumulator is full-sized (absolute index). float * __restrict vkq = VKQ_accum + jc_abs * (int64_t)DV; - const sycl::half * mask_h = nullptr; int64_t m_stride = 0; if (mask_data) { @@ -153,10 +162,8 @@ static void mkl_fa_online_softmax_chunk( mask_h = mask_data + (int64_t)m_head * mask_head_stride; m_stride = mask_row_stride; } - - // Row-wise local maximum (softcap before mask) - float local_max = -1e30f; - for (int i = 0; i < chunk_size; i++) { + // Score at chunk offset i — original per-element math. + auto score = [&](int i) { float s = KQ_row[i]; if (logit_softcap != 0.0f) { s = logit_softcap * sycl::tanh(s); @@ -165,40 +172,38 @@ static void mkl_fa_online_softmax_chunk( s += (float)mask_h[q_row * m_stride + (chunk_start + i)]; } + return s; + }; + // Pass 1: strided (coalesced) row-wise local maximum. + float local_max = -1e30f; + for (int i = local_id; i < chunk_size; i += local_size) { + float s = score(i); if (s > local_max) local_max = s; } - + const float final_local_max = sycl::reduce_over_group( + item.get_group(), local_max, sycl::maximum<float>()); // Rescale previous accumulator by exp(old_max - new_max) float old_max = KQ_max[jc_abs]; - float new_max = (old_max > local_max) ? old_max : local_max; + float new_max = (old_max > final_local_max) ? old_max : final_local_max; float rescale = (old_max < -1e29f) ? 1.0f : sycl::native::exp(old_max - new_max); - - for (int v = 0; v < DV; v++) { + for (int v = local_id; v < DV; v += local_size) { vkq[v] *= rescale; } - - // Softmax and write S_f16 (tile-local index) + // Pass 2: softmax numerators, strided; S row written once. float local_sum = 0.0f; - sycl::half * __restrict S_row = S_f16 - + jc_rel * (int64_t)chunk_size; - - for (int i = 0; i < chunk_size; i++) { - float s = KQ_row[i]; - if (logit_softcap != 0.0f) { - s = logit_softcap * sycl::tanh(s); - } - if (mask_h) { - s += (float)mask_h[q_row * m_stride - + (chunk_start + i)]; - } + for (int i = local_id; i < chunk_size; i += local_size) { + float s = score(i); float val = sycl::native::exp(s - new_max); S_row[i] = sycl::half(val); local_sum += val; } - - KQ_sum[jc_abs] = KQ_sum[jc_abs] * rescale + local_sum; - KQ_max[jc_abs] = new_max; + const float total_sum = sycl::reduce_over_group( + item.get_group(), local_sum, sycl::plus<float>()); + if (local_id == 0) { + KQ_sum[jc_abs] = KQ_sum[jc_abs] * rescale + total_sum; + KQ_max[jc_abs] = new_max; + } }); }); } diff --git a/ggml/src/ggml-sycl/fattn-tile.hpp b/ggml/src/ggml-sycl/fattn-tile.hpp index 9ba5296968d5..dcdcad88afab 100644 --- a/ggml/src/ggml-sycl/fattn-tile.hpp +++ b/ggml/src/ggml-sycl/fattn-tile.hpp @@ -1173,6 +1173,10 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_sycl_context & ctx, ggm launch_fattn_tile_switch_ncols1<DKQ, DV, 16, use_logit_softcap>(ctx, dst); return; } + if (use_gqa_opt && gqa_ratio % 8 == 0) { + launch_fattn_tile_switch_ncols1<DKQ, DV, 8, use_logit_softcap>(ctx, dst); + return; + } if (use_gqa_opt && gqa_ratio % 4 == 0) { launch_fattn_tile_switch_ncols1<DKQ, DV, 4, use_logit_softcap>(ctx, dst); return; diff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cpp index b5e79bea543d..d3e995233cf3 100644 --- a/ggml/src/ggml-sycl/fusion.cpp +++ b/ggml/src/ggml-sycl/fusion.cpp @@ -22,16 +22,22 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg const ggml_tensor * wg = gate->src[0]; const ggml_tensor * act = up->src[1]; - // one set of block offsets and one quantized activation must serve both weights - if (wu->type != wg->type || !ggml_are_same_shape(wu, wg) || !ggml_are_same_stride(wu, wg)) { + // one activation and one output indexing must serve both weights; the block types + // may differ, since the plain-layout fused kernel runs each operand's own vec_dot + // (different types then imply different byte strides, so only the shape must agree) + if (!ggml_are_same_shape(wu, wg)) { return false; } if (act != gate->src[1]) { return false; } - // only q4_K has a fused reorder GEMV so far, and it walks whole super-blocks - if (wu->type != GGML_TYPE_Q4_K || wu->ne[0] % QK_K != 0) { + // fused GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type + // q4_K, the plain-layout kernel covers q5_K / iq4_xs pairs incl. mixed gate/up types + const bool reorder_pair = wu->type == GGML_TYPE_Q4_K && wg->type == GGML_TYPE_Q4_K; + const bool plain_pair = (wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_IQ4_XS) && + (wg->type == GGML_TYPE_Q5_K || wg->type == GGML_TYPE_IQ4_XS); + if ((!reorder_pair && !plain_pair) || wu->ne[0] % QK_K != 0) { return false; } @@ -208,5 +214,67 @@ bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializ return true; } + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_SSM_CONV && ops.begin()[1] == GGML_OP_UNARY && + unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_SILU) { + const ggml_tensor * ssm_conv = cgraph->nodes[node_idx]; + const ggml_tensor * silu = cgraph->nodes[node_idx + 1]; + + if (ggml_get_unary_op(silu) != unary_ops.begin()[0]) { + return false; + } + if (ssm_conv->type != GGML_TYPE_F32 || silu->type != GGML_TYPE_F32) { + return false; + } + // the fused kernel writes the SiLU output with dense strides, so it must be contiguous + if (!ggml_is_contiguous(silu)) { + return false; + } + + return true; + } + + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_SSM_CONV && ops.begin()[1] == GGML_OP_ADD && + ops.begin()[2] == GGML_OP_UNARY && unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_SILU) { + const ggml_tensor * ssm_conv = cgraph->nodes[node_idx]; + const ggml_tensor * add = cgraph->nodes[node_idx + 1]; + const ggml_tensor * silu = cgraph->nodes[node_idx + 2]; + + if (ggml_get_unary_op(silu) != unary_ops.begin()[0]) { + return false; + } + if (ssm_conv->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 || silu->type != GGML_TYPE_F32) { + return false; + } + // the fused kernel writes the SiLU output with dense strides, so it must be contiguous + if (!ggml_is_contiguous(silu)) { + return false; + } + + // ADD must consume ssm_conv's output and broadcast a 1-D channel-wise bias + const ggml_tensor * bias = (add->src[0] == ssm_conv) ? add->src[1] : add->src[0]; + if (bias->type != GGML_TYPE_F32 || !ggml_is_contiguous(bias)) { + return false; + } + if (ggml_nelements(bias) != ssm_conv->ne[0] || bias->ne[0] != ssm_conv->ne[0]) { + return false; + } + + return true; + } + + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_SCALE) { + const ggml_tensor * rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor * scale = cgraph->nodes[node_idx + 1]; + GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + if (scale->src[0]->type != GGML_TYPE_F32 || scale->type != GGML_TYPE_F32) { + return false; + } + // the fused kernel reads/writes rows flat like the unfused pair + if (!ggml_is_contiguous_rows(rms_norm) || !ggml_is_contiguous_rows(scale)) { + return false; + } + return true; + } return false; } diff --git a/ggml/src/ggml-sycl/fwht.cpp b/ggml/src/ggml-sycl/fwht.cpp index 2312b3d131b7..39f273beaa9f 100644 --- a/ggml/src/ggml-sycl/fwht.cpp +++ b/ggml/src/ggml-sycl/fwht.cpp @@ -1,6 +1,50 @@ #include "fwht.hpp" #include <cmath> +#define P 1.0f +#define N -1.0f + +// constant Hadamard matrix via Paley I construction +static constexpr float H12[12][12] = { + { P, P, P, P, P, P, P, P, P, P, P, P }, + { P, N, P, N, P, P, P, N, N, N, P, N }, + { P, N, N, P, N, P, P, P, N, N, N, P }, + { P, P, N, N, P, N, P, P, P, N, N, N }, + { P, N, P, N, N, P, N, P, P, P, N, N }, + { P, N, N, P, N, N, P, N, P, P, P, N }, + { P, N, N, N, P, N, N, P, N, P, P, P }, + { P, P, N, N, N, P, N, N, P, N, P, P }, + { P, P, P, N, N, N, P, N, N, P, N, P }, + { P, P, P, P, N, N, N, P, N, N, P, N }, + { P, N, P, P, P, N, N, N, P, N, N, P }, + { P, P, N, P, P, P, N, N, N, P, N, N } +}; + +static constexpr float H20[20][20] = { + { P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P }, + { P, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N }, + { P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P }, + { P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P }, + { P, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N }, + { P, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N }, + { P, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N }, + { P, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N }, + { P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P }, + { P, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N }, + { P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P }, + { P, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N }, + { P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P }, + { P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P }, + { P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P }, + { P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P }, + { P, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N }, + { P, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N }, + { P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P }, + { P, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N } +}; + +#undef P +#undef N template <int N> static void fwht_kernel(const float * __restrict__ src, float * __restrict__ dst, const int64_t n_rows, @@ -80,6 +124,122 @@ static void launch_fwht(const float * src, float * dst, const int64_t n_rows, co }); } +template <int N, int m> +static void kronecker_kernel(const float * __restrict__ src, + float * __restrict__ dst, + const int64_t n_rows, + const float scale, + const sycl::nd_item<2> & item) { + static_assert(m == 12 || m == 20, "block size has to be 12 or 20."); + + const sycl::sub_group sg = item.get_sub_group(); + + const int64_t r = item.get_global_id(0); + if (r >= n_rows) { + return; + } + + src += r * N; + dst += r * N; + + constexpr int blocks_per_group = N / m; + constexpr int el_w = blocks_per_group / WARP_SIZE; + static_assert(el_w >= 1 && blocks_per_group % WARP_SIZE == 0, "blocks_per_group must be a multiple of WARP_SIZE"); + float reg[el_w * m]; + const int lane = sg.get_local_linear_id(); + +#pragma unroll + for (int i = 0; i < el_w; ++i) { + const int b_idx = i * WARP_SIZE + lane; + +#pragma unroll + for (int j = 0; j < m; ++j) { + reg[i * m + j] = src[b_idx * m + j] * scale; + } + } + +#pragma unroll + for (int b = 0; b < el_w; ++b) { + float z[m] = { 0.0f }; + +#pragma unroll + for (int i = 0; i < m; ++i) { +#pragma unroll + for (int j = 0; j < m; ++j) { + const float h = (m == 12 ? H12[j][i] : H20[j][i]); + z[i] += reg[b * m + j] * h; + } + } + +#pragma unroll + for (int i = 0; i < m; ++i) { + reg[b * m + i] = z[i]; + } + } + +#pragma unroll + for (int h = 1; h < WARP_SIZE; h *= 2) { +#pragma unroll + for (int j = 0; j < el_w; ++j) { +#pragma unroll + for (int k = 0; k < m; ++k) { + const float val = reg[j * m + k]; + const float val2 = dpct::permute_sub_group_by_xor(sg, val, h, WARP_SIZE); + + reg[j * m + k] = (lane & h) == 0 ? val + val2 : val2 - val; + } + } + } + +#pragma unroll + for (int h = WARP_SIZE; h < blocks_per_group; h *= 2) { + const int step = h / WARP_SIZE; +#pragma unroll + for (int j = 0; j < el_w; j += 2 * step) { +#pragma unroll + for (int s = 0; s < step; ++s) { +#pragma unroll + for (int k = 0; k < m; ++k) { + const float x = reg[(j + s) * m + k]; + const float y = reg[(j + s + step) * m + k]; + + reg[(j + s) * m + k] = x + y; + reg[(j + s + step) * m + k] = x - y; + } + } + } + } + +#pragma unroll + for (int i = 0; i < el_w; ++i) { + const int b_idx = i * WARP_SIZE + lane; +#pragma unroll + for (int k = 0; k < m; ++k) { + dst[b_idx * m + k] = reg[i * m + k]; + } + } +} + +template <int N, int m> +static void launch_kronecker(const float * src, + float * dst, + const int64_t n_rows, + const float scale, + dpct::queue_ptr stream) { + constexpr int rows_per_block = 4; + + const int64_t num_blocks = (n_rows + rows_per_block - 1) / rows_per_block; + + // dim 1 is the fastest-varying, so a sub-group is exactly one row's WARP_SIZE lanes. + const sycl::range<2> global(num_blocks * rows_per_block, WARP_SIZE); + const sycl::range<2> local(rows_per_block, WARP_SIZE); + + stream->parallel_for(sycl::nd_range<2>(global, local), + [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + kronecker_kernel<N, m>(src, dst, n_rows, scale, item); + }); +} + bool ggml_sycl_op_fwht(ggml_backend_sycl_context & ctx, const ggml_tensor * src, ggml_tensor * dst) { if (src->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { return false; @@ -113,6 +273,18 @@ bool ggml_sycl_op_fwht(ggml_backend_sycl_context & ctx, const ggml_tensor * src, case 512: launch_fwht<512>(src_d, dst_d, rows, scale, stream); return true; + case 384: + launch_kronecker<384, 12>(src_d, dst_d, rows, scale, stream); + return true; + case 768: + launch_kronecker<768, 12>(src_d, dst_d, rows, scale, stream); + return true; + case 640: + launch_kronecker<640, 20>(src_d, dst_d, rows, scale, stream); + return true; + case 1280: + launch_kronecker<1280, 20>(src_d, dst_d, rows, scale, stream); + return true; default: return false; } diff --git a/ggml/src/ggml-sycl/gemm.hpp b/ggml/src/ggml-sycl/gemm.hpp index c202da110beb..81bc5c2e6b47 100644 --- a/ggml/src/ggml-sycl/gemm.hpp +++ b/ggml/src/ggml-sycl/gemm.hpp @@ -66,8 +66,10 @@ class DnnlGemmWrapper { auto matmul_pd = dnnl::matmul::primitive_desc(eng, a_in_md, b_in_md, c_md, primitive_attr); auto c_mem = dnnl::memory(matmul_pd.dst_desc(), eng, c); - auto scratchpad_md = matmul_pd.scratchpad_desc(); - auto scratchpad_mem = ctx.get_scratchpad_mem(scratchpad_md, eng, q); + const auto scratchpad_md = matmul_pd.scratchpad_desc(); + ggml_sycl_pool_alloc<uint8_t> scratchpad(ctx.pool()); + void * scratchpad_ptr = scratchpad_md.get_size() > 0 ? scratchpad.alloc(scratchpad_md.get_size()) : nullptr; + auto scratchpad_mem = dnnl::memory(scratchpad_md, eng, scratchpad_ptr); auto matmul_prim = dnnl::matmul(matmul_pd); diff --git a/ggml/src/ggml-sycl/getrows.cpp b/ggml/src/ggml-sycl/getrows.cpp index 2113f3563398..4e84bd22253b 100644 --- a/ggml/src/ggml-sycl/getrows.cpp +++ b/ggml/src/ggml-sycl/getrows.cpp @@ -245,6 +245,84 @@ static void get_rows_sycl_float(ggml_backend_sycl_context & ctx, const ggml_tens GGML_UNUSED(ctx); } +template <typename src0_t> +static void k_get_rows_back_float(const src0_t * src0, const int32_t * src1, float * dst, + const int64_t ncols, const int64_t nrows_grad_10, const int64_t nrows_grad_11, const int64_t nrows_dst, + const size_t s01, const size_t s02, + const size_t s10, const size_t s11, + const size_t s1, + const int64_t block_num_y, + const sycl::nd_item<3> & item_ct1) { + const int64_t col = item_ct1.get_group(2) * item_ct1.get_local_range(2) + item_ct1.get_local_id(2); + if (col >= ncols) { + return; + } + + // block_num_y is clamped, so stride over destination rows like CUDA k_get_rows_back_float + for (int64_t dst_row = item_ct1.get_group(1); dst_row < nrows_dst; dst_row += block_num_y) { + float sum = 0.0f; + + const int64_t nrows_grad_total = nrows_grad_10 * nrows_grad_11; + for (int64_t i = 0; i < nrows_grad_total; ++i) { + const int64_t i10 = i % nrows_grad_10; + const int64_t i11 = i / nrows_grad_10; + if (src1[i10*s10 + i11*s11] != dst_row) { + continue; + } + sum += (float) src0[col + i10*s01 + i11*s02]; + } + + dst[col + dst_row*s1] = sum; + } +} + +template <typename src0_t> +static void get_rows_back_sycl_float(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst, + const src0_t * src0_dd, const int32_t * src1_dd, + float * dst_dd, queue_ptr stream) { + + GGML_TENSOR_BINARY_OP_LOCALS + + GGML_ASSERT(ne02*ne03 == 1); + GGML_ASSERT(ne12*ne13 == 1); + GGML_ASSERT(ne2*ne3 == 1); + GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); + GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type)); + GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type)); + + const int64_t ncols = ne00; + const int64_t nrows_grad_10 = ne10; + const int64_t nrows_grad_11 = ne11; + const int64_t nrows_dst = ne1; + + const size_t s01 = nb01 / sizeof(src0_t); + const size_t s02 = nb02 / sizeof(src0_t); + + const size_t s10 = nb10 / sizeof(int32_t); + const size_t s11 = nb11 / sizeof(int32_t); + + const size_t s1 = nb1 / sizeof(float); + + const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE); + const int64_t block_num_x = (ncols + SYCL_GET_ROWS_BLOCK_SIZE - 1) / SYCL_GET_ROWS_BLOCK_SIZE; + const int64_t block_num_y = std::min<int64_t>(nrows_dst, (int64_t) UINT16_MAX); + const sycl::range<3> block_nums(1, block_num_y, block_num_x); + + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_get_rows_back_float(src0_dd, src1_dd, dst_dd, + ncols, nrows_grad_10, nrows_grad_11, nrows_dst, + s01, s02, s10, s11, s1, + block_num_y, item_ct1); + }); + + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(ctx); +} + void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { GGML_ASSERT(dst->src[1]->type == GGML_TYPE_I32); GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_I32 ); @@ -366,3 +444,30 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { GGML_ABORT("fatal error"); } } + +void ggml_sycl_op_get_rows_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + GGML_ASSERT(src1->type == GGML_TYPE_I32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_ASSERT(ggml_is_contiguous(dst)); + + switch (src0->type) { + case GGML_TYPE_F16: + get_rows_back_sycl_float(ctx, src0, src1, dst, (const sycl::half *) src0->data, + (const int32_t *) src1->data, (float *) dst->data, + ctx.stream()); + break; + case GGML_TYPE_F32: + get_rows_back_sycl_float(ctx, src0, src1, dst, (const float *) src0->data, + (const int32_t *) src1->data, (float *) dst->data, + ctx.stream()); + break; + default: + GGML_ABORT("%s: unsupported src0 type: %s\n", __func__, ggml_type_name(src0->type)); + break; + } +} diff --git a/ggml/src/ggml-sycl/getrows.hpp b/ggml/src/ggml-sycl/getrows.hpp index 1c560cd9f894..0388e6c7ffbe 100644 --- a/ggml/src/ggml-sycl/getrows.hpp +++ b/ggml/src/ggml-sycl/getrows.hpp @@ -16,5 +16,6 @@ #include "common.hpp" void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor *dst); +void ggml_sycl_op_get_rows_back(ggml_backend_sycl_context & ctx, ggml_tensor *dst); #endif // GGML_SYCL_GETROWS_HPP diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 27804e07301d..e13ec852511c 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -91,12 +91,15 @@ static bool g_sycl_loaded = false; int g_ggml_sycl_debug = 0; +int g_ggml_sycl_dev_debug = 0; int g_ggml_sycl_enable_optimize = 1; int g_ggml_sycl_enable_graph = 0; int g_ggml_sycl_enable_dnn = 1; int g_ggml_sycl_fa_onednn = 1; int g_ggml_sycl_fa_onednn_max_kv = 0; int g_ggml_sycl_enable_mkl_fa = 1; +int g_ggml_sycl_memtrace = 0; +int g_ggml_sycl_memtrace_step = 64; int g_ggml_sycl_enable_vmm = 1; int g_ggml_sycl_enable_fusion = 1; int g_ggml_sycl_enable_esimd = 1; @@ -111,8 +114,8 @@ int g_ggml_sycl_enable_host_pinned_mem = 1; int g_ggml_sycl_host_pinned_mem_2g = 0; int g_ggml_sycl_get_mem_api = MEMORY_API_TYPE_LEVEL_ZERO; - static ggml_sycl_device_info ggml_sycl_init() { + GGML_SYCL_DEBUG("[SYCL] call ggml_sycl_init\n"); ggml_sycl_device_info info = {}; // Do not hard crash when there exists no SYCL devices. @@ -203,12 +206,9 @@ static ggml_sycl_device_info ggml_sycl_init() { } #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API - // Large buffers can be allocated before ggml_check_sycl() initializes other - // g_ggml_sycl_enable_* globals, so initialize this one as early as we can. + //update g_ggml_sycl_use_level_zero_api according to the device support g_ggml_sycl_use_level_zero_api = - info.ext_oneapi_level_zero && ggml_sycl_get_env("GGML_SYCL_USE_LEVEL_ZERO_API", 1); -#else - g_ggml_sycl_use_level_zero_api = 0; + info.ext_oneapi_level_zero && g_ggml_sycl_use_level_zero_api; #endif return info; @@ -312,37 +312,60 @@ static const char* dev2dev_int2str(int dev2dev) { * It's the first internal function to be called by them in SYCL backend. * This function is used to do initialize work for the SYCL backend and set the global variables. */ -void initialize_sycl_begining() { #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API - ze_result_t zes_init = zesInit(0); - if (zes_init != ZE_RESULT_SUCCESS) { - std::cerr << "Warning: zesInit failed [ggml_check_sycl] with code " << static_cast<int>(zes_init) - << ". Sysman free-memory query may be unavailable.\n"; +static ze_result_t init_zes() { + ze_result_t res = zesInit(0); + if (res != ZE_RESULT_SUCCESS) { + GGML_SYCL_DEBUG("Warning: [%s] zesInit failed with code %d. Sysman free-memory query be unavailable.\n", + __func__, (int) res); } + return res; +} + +ze_result_t get_zes_init_res() { + static ze_result_t zes_init_res = init_zes(); + GGML_SYCL_DEBUG("[SYCL] call %s: zesInit result: %d\n", __func__, (int) zes_init_res); + return zes_init_res; +} +#endif + +void initialize_sycl_begining() { +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + //must be called in initialization stage, before any other Level Zero API calls + GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); + get_zes_init_res(); #endif } static void ggml_check_sycl() try { + GGML_SYCL_DEBUG("[SYCL] ggml_check_sycl()\n"); static bool initialized = false; if (!initialized) { initialize_sycl_begining(); g_ggml_sycl_debug = ggml_sycl_get_env("GGML_SYCL_DEBUG", 0); + g_ggml_sycl_dev_debug = ggml_sycl_get_env("GGML_SYCL_DEV_DEBUG", 0); g_ggml_sycl_enable_optimize = ggml_sycl_get_env("GGML_SYCL_ENABLE_OPT", 1); g_ggml_sycl_enable_graph = ggml_sycl_get_env("GGML_SYCL_ENABLE_GRAPH", 0); g_ggml_sycl_enable_dnn = ggml_sycl_get_env("GGML_SYCL_ENABLE_DNN", 1); g_ggml_sycl_fa_onednn = ggml_sycl_get_env("GGML_SYCL_FA_ONEDNN", 1); g_ggml_sycl_fa_onednn_max_kv = ggml_sycl_get_env("GGML_SYCL_FA_ONEDNN_MAX_KV", 0); g_ggml_sycl_enable_mkl_fa = ggml_sycl_get_env("GGML_SYCL_ENABLE_MKL_FA", 1); + g_ggml_sycl_memtrace = ggml_sycl_get_env("GGML_SYCL_MEMTRACE", 0); + g_ggml_sycl_memtrace_step = ggml_sycl_get_env("GGML_SYCL_MEMTRACE_STEP", 64); g_ggml_sycl_enable_vmm = ggml_sycl_get_env("GGML_SYCL_ENABLE_VMM", 1); g_ggml_sycl_enable_fusion = ggml_sycl_get_env("GGML_SYCL_ENABLE_FUSION", 1); g_ggml_sycl_enable_esimd = ggml_sycl_get_env("GGML_SYCL_ENABLE_ESIMD", 1); g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0); +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + g_ggml_sycl_use_level_zero_api = ggml_sycl_get_env("GGML_SYCL_USE_LEVEL_ZERO_API", 1); +#else + g_ggml_sycl_use_level_zero_api = 0; +#endif g_ggml_sycl_dev2dev_memcpy = ggml_sycl_get_env("GGML_SYCL_DEV2DEV_MEMCPY", DEV2DEV_MEMCPY_SYCL); g_ggml_sycl_get_mem_api = ggml_sycl_get_env("GGML_SYCL_GET_MEM_API", MEMORY_API_TYPE_LEVEL_ZERO); - if (g_ggml_sycl_use_level_zero_api == 0) { g_ggml_sycl_dev2dev_memcpy = DEV2DEV_MEMCPY_SYCL; g_ggml_sycl_get_mem_api = MEMORY_API_TYPE_SYCL; @@ -401,6 +424,7 @@ static void ggml_check_sycl() try { GGML_LOG_INFO("Running with Environment Variables:\n"); GGML_LOG_INFO(" GGML_SYCL_DEBUG: %d\n", g_ggml_sycl_debug); + GGML_LOG_INFO(" GGML_SYCL_DEV_DEBUG: %d\n", g_ggml_sycl_dev_debug); #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d (%s)\n", g_ggml_sycl_dev2dev_memcpy, dev2dev_int2str(g_ggml_sycl_dev2dev_memcpy)); @@ -421,6 +445,8 @@ static void ggml_check_sycl() try { #endif GGML_LOG_INFO(" GGML_SYCL_FA_ONEDNN_MAX_KV: %d\n", g_ggml_sycl_fa_onednn_max_kv); GGML_LOG_INFO(" GGML_SYCL_ENABLE_MKL_FA: %d\n", g_ggml_sycl_enable_mkl_fa); + GGML_LOG_INFO(" GGML_SYCL_MEMTRACE: %d\n", g_ggml_sycl_memtrace); + GGML_LOG_INFO(" GGML_SYCL_MEMTRACE_STEP: %d\n", g_ggml_sycl_memtrace_step); #ifdef SYCL_FLASH_ATTN GGML_LOG_INFO(" GGML_SYCL_ENABLE_FLASH_ATTN: %d\n", g_ggml_sycl_enable_flash_attention); #else @@ -939,6 +965,7 @@ inline void * aligned_malloc_host(size_t alignment, size_t size) { static ggml_backend_buffer_t ggml_backend_sycl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) try { + GGML_SYCL_DEBUG("[SYCL] call %s: size=%zu\n", __func__, size); ggml_check_sycl(); ggml_backend_sycl_buffer_type_context * buft_ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; @@ -964,7 +991,7 @@ ggml_backend_sycl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, return nullptr; } } else { - SYCL_CHECK(CHECK_TRY_ERROR(dev_ptr = (void *)ggml_sycl_malloc_device(size, *stream))); + SYCL_CHECK(CHECK_TRY_ERROR(dev_ptr = (void *)ggml_sycl_malloc_device(size, *stream, GGML_SYCL_MEM_BUFFER))); if (!dev_ptr) { GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on device\n", __func__, size); return nullptr; @@ -984,12 +1011,25 @@ static size_t ggml_backend_sycl_buffer_type_get_alignment(ggml_backend_buffer_ty GGML_UNUSED(buft); } +bool is_bmg_g31_arch(int device) { + return ggml_sycl_info().devices[device].hw_info.arch == gpu_arch::intel_gpu_bmg_g31; +} + static size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { size_t max_alloc_size = dpct::get_current_device().get_max_mem_alloc_size(); if (g_ggml_sycl_host_pinned_mem_2g) { return std::min(max_alloc_size, (size_t) 2LL*1024*1024*1024); } else { + ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; + int device = ctx->device; + if(is_bmg_g31_arch(device)) { + //Todo, it's workaround for BMG-G31, which has a known issue with large allocations. + //The max alloc size is reduced to 60% of the reported max alloc size. + //remove it after https://github.com/intel/compute-runtime/issues/998 is fixed. + max_alloc_size = max_alloc_size*0.6; + } return max_alloc_size; + } GGML_UNUSED(buft); } @@ -1217,7 +1257,7 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_sycl_set_device(i); const queue_ptr stream = ctx->streams[i]; char * buf; - SYCL_CHECK(CHECK_TRY_ERROR(buf = (char *)ggml_sycl_malloc_device(size, *stream))); + SYCL_CHECK(CHECK_TRY_ERROR(buf = (char *)ggml_sycl_malloc_device(size, *stream, GGML_SYCL_MEM_BUFFER))); if (!buf) { char err_buf[1024]; snprintf(err_buf, 1023, "%s: can't allocate %zu Bytes of memory on device\n", __func__, size); @@ -1457,11 +1497,14 @@ static ggml_backend_buffer_type_i ggml_backend_sycl_split_buffer_type_interface /* .is_host = */ ggml_backend_sycl_split_buffer_type_is_host, }; -ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split) { +ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(int main_device, const float * tensor_split) { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_split_buffer_type\n"); + + GGML_UNUSED(main_device); + static std::mutex mutex; std::lock_guard<std::mutex> lock(mutex); - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_split_buffer_type\n"); ggml_check_sycl(); // FIXME: this is not thread safe static std::map<std::array<float, GGML_SYCL_MAX_DEVICES>, struct ggml_backend_buffer_type> buft_map; @@ -1512,13 +1555,18 @@ static const char * ggml_backend_sycl_host_buffer_type_name(ggml_backend_buffer_ GGML_UNUSED(buft); } +static int ggml_backend_sycl_host_buffer_type_device(ggml_backend_buffer_type_t buft) { + return static_cast<const ggml_backend_sycl_device_context *>(buft->device->context)->device; +} + //host pinned memory -static void * ggml_backend_sycl_host_malloc(size_t size) { +static void * ggml_backend_sycl_host_malloc(int device, size_t size) { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_host_malloc of size %.2f MiB on device %d\n", size / 1024.0 / 1024.0, device); void * ptr = nullptr; try { ggml_check_sycl(); // USM host memory is page-locked and device-accessible by construction - auto & q = dpct::dev_mgr::instance().get_device(0).default_queue(); + auto & q = dpct::dev_mgr::instance().get_device(device).default_queue(); ptr = sycl::malloc_host(size, q, sycl::property_list{}); } catch (...) { ptr = nullptr; @@ -1536,7 +1584,8 @@ static void ggml_backend_sycl_host_buffer_free_buffer(ggml_backend_buffer_t buff return; } if (g_ggml_sycl_enable_host_pinned_mem) { - auto & q = dpct::dev_mgr::instance().get_device(0).default_queue(); + const int device = ggml_backend_sycl_host_buffer_type_device(buffer->buft); + auto & q = dpct::dev_mgr::instance().get_device(device).default_queue(); SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(buffer->context, q))); } else { free_aligned_mem_host((void *) buffer->context); @@ -1544,8 +1593,9 @@ static void ggml_backend_sycl_host_buffer_free_buffer(ggml_backend_buffer_t buff } static ggml_backend_buffer_t ggml_backend_sycl_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { - void * ptr = g_ggml_sycl_enable_host_pinned_mem ? ggml_backend_sycl_host_malloc(size) : - aligned_malloc_host(TENSOR_ALIGNMENT, size); + void * ptr = g_ggml_sycl_enable_host_pinned_mem ? + ggml_backend_sycl_host_malloc(ggml_backend_sycl_host_buffer_type_device(buft), size) : + aligned_malloc_host(TENSOR_ALIGNMENT, size); if (ptr == nullptr) { // fallback to cpu buffer return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size); @@ -1562,8 +1612,8 @@ static ggml_backend_buffer_t ggml_backend_sycl_host_buffer_type_alloc_buffer(ggm static size_t ggml_backend_sycl_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { if (g_ggml_sycl_enable_host_pinned_mem) { - ggml_backend_sycl_device_context * dev_ctx = (ggml_backend_sycl_device_context *) buft->device->context; - size_t max_alloc_size = dpct::dev_mgr::instance().get_device(dev_ctx->device).get_max_mem_alloc_size(); + const int device = ggml_backend_sycl_host_buffer_type_device(buft); + size_t max_alloc_size = dpct::dev_mgr::instance().get_device(device).get_max_mem_alloc_size(); if (g_ggml_sycl_host_pinned_mem_2g) { return std::min(max_alloc_size, (size_t) 2LL*1024*1024*1024); } else { @@ -1574,22 +1624,37 @@ static size_t ggml_backend_sycl_host_buffer_type_get_max_size(ggml_backend_buffe } } -ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type() { - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_host_buffer_type\n"); - static struct ggml_backend_buffer_type ggml_backend_sycl_buffer_type_host = { - /* .iface = */ { - /* .get_name = */ ggml_backend_sycl_host_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_sycl_host_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, - /* .get_max_size = */ ggml_backend_sycl_host_buffer_type_get_max_size, - /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, - /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, - }, - /* .device = */ ggml_backend_reg_dev_get(ggml_backend_sycl_reg(), 0), - /* .context = */ nullptr, - }; +static ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type_for_device(int device) { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_host_buffer_type_for_device on device %d\n", device); - return &ggml_backend_sycl_buffer_type_host; + // the vector is never resized after this, so the returned pointers stay valid + static std::vector<ggml_backend_buffer_type> buffer_types_host = [] { + std::vector<ggml_backend_buffer_type> bufts(ggml_backend_sycl_get_device_count()); + for (size_t i = 0; i < bufts.size(); i++) { + bufts[i] = { + /* .iface = */ { + /* .get_name = */ ggml_backend_sycl_host_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_sycl_host_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, + /* .get_max_size = */ ggml_backend_sycl_host_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, + /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, + }, + /* .device = */ ggml_backend_reg_dev_get(ggml_backend_sycl_reg(), i), + /* .context = */ nullptr, + }; + } + return bufts; + }(); + + GGML_ASSERT(device >= 0 && device < (int) buffer_types_host.size()); + + return &buffer_types_host[device]; +} + +// TODO: this function is unused and is a temporary hack to avoid breaking changes +ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type() { + return ggml_backend_sycl_host_buffer_type_for_device(0); } // buffer pool for sycl (legacy) @@ -1697,7 +1762,7 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool { void * ptr; size_t look_ahead_size = (size_t) (1.05 * size); - SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *)ggml_sycl_malloc_device(look_ahead_size, *qptr))); + SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *)ggml_sycl_malloc_device(look_ahead_size, *qptr, GGML_SYCL_MEM_POOL_LEG))); if (!ptr) { GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on device/GPU\n", __func__, look_ahead_size); return nullptr; @@ -1786,6 +1851,13 @@ struct ggml_sycl_pool_vmm : public ggml_sycl_pool { GGML_ASSERT(pool_size + reserve_size <= SYCL_POOL_VMM_MAX_SIZE); + if (ggml_sycl_memtrace_enabled()) { + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " pool_vmm[%d] committing %5zu MiB (pool %5zu -> %5zu MiB)\n", + device, reserve_size / (1024 * 1024), pool_size / (1024 * 1024), + (pool_size + reserve_size) / (1024 * 1024)); + ggml_sycl_memtrace_report("before pool_vmm commit"); + } + // allocate more physical memory std::optional<sycl::ext::oneapi::experimental::physical_mem> phys; SYCL_CHECK(CHECK_TRY_ERROR(phys.emplace(dev, ctx, reserve_size))); @@ -1811,6 +1883,7 @@ struct ggml_sycl_pool_vmm : public ggml_sycl_pool { // add to the pool pool_size += reserve_size; + ggml_sycl_memtrace_add(GGML_SYCL_MEM_POOL_VMM, map_ptr, reserve_size); #ifdef DEBUG_SYCL_MALLOC GGML_LOG_INFO("sycl pool[%d]: size increased to %llu MB (reserved %llu MB)\n", @@ -3084,10 +3157,14 @@ static void ggml_sycl_op_top_k(ggml_backend_sycl_context & ctx, ggml_tensor * ds const int64_t ncols = src0->ne[0]; const int64_t nrows = ggml_nrows(src0); - GGML_ASSERT(k > 0 && k <= 32); + GGML_ASSERT(k > 0); GGML_ASSERT(k <= ncols); - top_k_f32_sycl(ctx, src0_dd, dst_dd, ncols, nrows, k, main_stream); + if (k <= SYCL_TOP_K_SCAN_MERGE_MAX_K) { + top_k_f32_sycl(ctx, src0_dd, dst_dd, ncols, nrows, k, main_stream); + } else { + ggml_sycl_top_k_radix(ctx, src0_dd, dst_dd, ncols, nrows, k, main_stream); + } } inline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { @@ -3202,7 +3279,7 @@ inline void ggml_sycl_op_scale(ggml_backend_sycl_context & ctx, ggml_tensor * ds SYCL_CHECK(0); } -static void ggml_sycl_set_peer_access(const int n_tokens, int main_device) { +static void ggml_sycl_set_peer_access(const int n_tokens, [[maybe_unused]] int main_device) { static bool peer_access_enabled = false; const bool enable_peer_access = n_tokens <= GGML_SYCL_PEER_MAX_BATCH_SIZE; @@ -3263,7 +3340,6 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten GGML_ASSERT(!ggml_backend_buffer_is_sycl_split(dst->buffer)); GGML_ASSERT(!ggml_backend_buffer_is_sycl_split(src1->buffer)); - GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1)); GGML_ASSERT(ne12 >= ne02 && ne12 % ne02 == 0); @@ -3431,7 +3507,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten // for split tensors the data begins at i0 == i0_offset_low char * src0_dd_i = dev[i].src0_dd + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs; - float * src1_ddf_i = dev[i].src1_ddf + (i0*ne11 + src1_col_0) * ne10; + float * src1_ddf_i = (float *) ((char *) dev[i].src1_ddf + + (i0*ne11 + src1_col_0) * ne10 * ggml_type_size(src1->type)); char * src1_ddq_i = dev[i].src1_ddq + src1_ddq_i_offset; float * dst_dd_i = dev[i].dst_dd + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff); @@ -3452,12 +3529,12 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten src1_ncols * src1_padded_col_size * q8_1_ts / q8_1_bs) .wait())); } else { - float * src1_ddf_i_source = (float *) src1_extra->data_device[ctx.device]; - src1_ddf_i_source += (i0 * ne11 + src1_col_0) * ne10; + const char * src1_ddf_i_source = (const char *) src1_extra->data_device[ctx.device] + + (i0 * ne11 + src1_col_0) * ne10 * ggml_type_size(src1->type); SYCL_CHECK( CHECK_TRY_ERROR(dev2dev_memcpy(i, *stream, ctx.device, *main_stream, src1_ddf_i, src1_ddf_i_source, - src1_ncols * ne10 * sizeof(float)))); + src1_ncols * ne10 * ggml_type_size(src1->type)))); } } } else { @@ -3563,6 +3640,11 @@ static void ggml_sycl_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * ds ggml_sycl_op_get_rows(ctx, dst); } +static void ggml_sycl_get_rows_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + ggml_sycl_op_get_rows_back(ctx, dst); +} + static void ggml_sycl_norm(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); ggml_sycl_op_norm(ctx, dst); @@ -4039,7 +4121,9 @@ static inline void * sycl_ext_malloc_device(dpct::queue_ptr stream, size_t size) bool use_async = g_ggml_sycl_use_async_mem_op; #if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC if (use_async) { - return syclex::async_malloc(*stream, sycl::usm::alloc::device, size); + void * ptr = syclex::async_malloc(*stream, sycl::usm::alloc::device, size); + ggml_sycl_memtrace_add(GGML_SYCL_MEM_ASYNC, ptr, size); + return ptr; } #else // If async allocation extension is not available, use_async should always be false. @@ -4052,6 +4136,7 @@ static inline void sycl_ext_free(dpct::queue_ptr stream, void * ptr) { bool use_async = g_ggml_sycl_use_async_mem_op; #if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC if (use_async) { + ggml_sycl_memtrace_del(ptr); syclex::async_free(*stream, ptr); return; } @@ -4780,6 +4865,43 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor } } +// {mul_mat(gate), mul_mat(up), GLU} over the standard (non-reorder) weight layout, +// for quant pairs the reorder kernel does not cover (mixed gate/up types, e.g. UD-Q4_K_XL's +// iq4_xs gate + q5_K up). Two launches replace five: one shared q8_1 quantization and one +// dual-GEMV+GLU. +static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, ggml_tensor * glu, + ggml_tensor * gate, ggml_tensor * up, const ggml_tensor * wu, + const ggml_tensor * wg, const ggml_tensor * act) { + // weights already migrated to the reorder layout would be misread by the plain kernel + const auto * extra_u = static_cast<const ggml_tensor_extra_gpu *>(wu->extra); + const auto * extra_g = static_cast<const ggml_tensor_extra_gpu *>(wg->extra); + if ((extra_u && extra_u->optimized_feature.reorder) || (extra_g && extra_g->optimized_feature.reorder)) { + return false; + } + + // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises + scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (plain layout)"); + + const int64_t ne00 = wu->ne[0]; + const int64_t ne11 = act->ne[1]; + + const queue_ptr stream = ctx.stream(); + const int src1_padded_cols = GGML_PAD((int) ne00, MATRIX_ROW_PADDING); + + ggml_sycl_pool_alloc<char> src1_q8_alloc(ctx.pool(), + (size_t) ne11 * src1_padded_cols * sizeof(block_q8_1) / QK8_1); + char * src1_ddq = src1_q8_alloc.get(); + + quantize_row_q8_1_sycl<quantize_q8_1>((const float *) act->data, src1_ddq, (int) ne00, (int) ne11, + src1_padded_cols, stream); + + return ggml_sycl_mul_mat_vec_q_glu_plain(wg->type, wu->type, ggml_get_glu_op(glu), wg->data, wu->data, + src1_ddq, (float *) glu->data, (int) ne00, (int) wu->ne[1], + (int) ne11, + /*stride_col_y=*/src1_padded_cols / QK8_1, + /*stride_col_dst=*/(int) glu->ne[0], stream); +} + // Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx. // Returns false if it declined, in which case the caller runs the three nodes normally. static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) { @@ -4805,6 +4927,12 @@ static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, gg return false; } + // quant pairs the reorder kernel cannot serve (mixed gate/up types) take the + // standard-layout fused path instead; q4_K keeps the reorder path below + if (wg->type != GGML_TYPE_Q4_K || wu->type != GGML_TYPE_Q4_K) { + return ggml_sycl_mul_mat_glu_mmvq_plain(ctx, glu, gate, up, wu, wg, act); + } + // install the reorder (SoA) layout the fused kernel needs, as the unfused mmvq path would; // a no-op once done. after the bail checks so a declined op does not pay for it. opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::MMVQ); @@ -4841,6 +4969,78 @@ static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, gg /*stride_col_dst=*/(int) glu->ne[0], stream); } +// Batch the run of consecutive L2_NORM siblings starting at node_idx into one launch. +// Returns the number of extra graph nodes consumed, or 0 if the run is shorter than two +// (the caller then runs the norm through the per-tensor kernel). +static int ggml_sycl_l2_norm_batch_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) { + const ggml_tensor * node = cgraph->nodes[node_idx]; + if (ggml_sycl_info().device_count != 1 || node->type != GGML_TYPE_F32 || + node->src[0]->type != GGML_TYPE_F32 || node->src[0]->ne[0] >= 1024) { + return 0; + } + + ggml_tensor * batch[GGML_SYCL_L2_BATCH_MAX]; + int count = 0; + int last = node_idx; + float eps0; + memcpy(&eps0, node->op_params, sizeof(float)); + + // Conservative aliasing test: the batched norms run concurrently in one kernel, + // so none may read what another writes, and none may write where another writes. + auto overlaps = [](const ggml_tensor * a, const ggml_tensor * b) { + const char * ab = (const char *) a->data; + const char * bb = (const char *) b->data; + return ab < bb + ggml_nbytes(b) && bb < ab + ggml_nbytes(a); + }; + + for (int j = node_idx; j < cgraph->n_nodes && count < GGML_SYCL_L2_BATCH_MAX; ++j) { + ggml_tensor * nj = cgraph->nodes[j]; + if (ggml_is_empty(nj) || nj->op == GGML_OP_RESHAPE || nj->op == GGML_OP_TRANSPOSE || + nj->op == GGML_OP_VIEW || nj->op == GGML_OP_PERMUTE || nj->op == GGML_OP_NONE || + (nj->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + continue; // not a launch; cannot break a run of adjacent norms + } + if (nj->op != GGML_OP_L2_NORM || nj->type != GGML_TYPE_F32 || + nj->src[0]->type != GGML_TYPE_F32 || !ggml_are_same_shape(nj, node) || + !ggml_are_same_shape(nj->src[0], node->src[0])) { + break; // any other launch ends the run + } + bool same_nb = true; + for (int d = 0; d < GGML_MAX_DIMS; ++d) { + if (nj->nb[d] != node->nb[d] || nj->src[0]->nb[d] != node->src[0]->nb[d]) { + same_nb = false; + break; + } + } + if (!same_nb) { + break; // one nb[] stride set is shared by the whole batch + } + float epsj; + memcpy(&epsj, nj->op_params, sizeof(float)); + if (epsj != eps0) { + break; // eps mismatch ends the run + } + bool indep = true; + for (int k = 0; k < count; ++k) { + if (overlaps(nj->src[0], batch[k]) || overlaps(nj, batch[k])) { + indep = false; + break; + } + } + if (!indep) { + break; // an overlapping tensor would race inside one launch + } + batch[count++] = nj; + last = j; + } + if (count < 2) { + return 0; // a lone norm falls through to the per-tensor kernel + } + ggml_sycl_l2_norm_batch(ctx, batch, count); + return last - node_idx; +} + + __dpct_inline__ static void k_copy_src1_to_contiguous( const char *__restrict__ src1_original, char *__restrict__ src1_contiguous, const mmid_row_mapping *__restrict__ row_mapping, @@ -5252,8 +5452,8 @@ catch (sycl::exception const &exc) { } static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggml_tensor * dst) try { + GGML_SYCL_DEBUG("[SYCL] ggml_sycl_compute_forward: dst=%s, op=%s\n", dst->name, ggml_op_name(dst->op)); if (!g_sycl_loaded) return false; - initialize_sycl_begining(); if (dst->src[0] != nullptr && ggml_backend_buffer_is_sycl_split(dst->src[0]->buffer)) { ggml_sycl_set_peer_access(dst->src[1]->ne[1], ctx.device); @@ -5287,6 +5487,9 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_GET_ROWS: ggml_sycl_get_rows(ctx, dst); break; + case GGML_OP_GET_ROWS_BACK: + ggml_sycl_get_rows_back(ctx, dst); + break; case GGML_OP_SET: ggml_sycl_op_set(ctx, dst); break; @@ -5636,13 +5839,30 @@ catch (sycl::exception const &exc) { std::exit(1); } +bool sycl_get_mem_info(int device, size_t * free, size_t * total) { + GGML_SYCL_DEBUG("[SYCL] [%s] g_ggml_sycl_get_mem_api=%d\n", + __func__, g_ggml_sycl_get_mem_api); + + MemoryAPIType mem_api_type = MemoryAPIType::MEMORY_API_TYPE_SYCL; + +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + mem_api_type = get_zes_init_res() == ZE_RESULT_SUCCESS ? + (MemoryAPIType) g_ggml_sycl_get_mem_api : MemoryAPIType::MEMORY_API_TYPE_SYCL; +#else + mem_api_type = MemoryAPIType::MEMORY_API_TYPE_SYCL; +#endif + bool res = get_memory_size(dpct::dev_mgr::instance().get_device(device), + *free, *total, mem_api_type); + GGML_SYCL_DEBUG("[SYCL] [%s] total = %zu free = %zu\n", __func__, *total, *free); + return res; +} + void ggml_backend_sycl_get_device_memory(int device, size_t * free, size_t * total) try { GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_get_device_memory\n"); - bool res = get_memory_size(dpct::dev_mgr::instance().get_device(device), *free, *total, - (MemoryAPIType) g_ggml_sycl_get_mem_api); - if (!res) { + if (!sycl_get_mem_info(device, free, total)) { GGML_ABORT("[%s] failed to get device memory size", __func__); } + ggml_sycl_memtrace_report_device("device memory query", device, *free, *total); } catch (const sycl::exception & exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ << ", line:" << __LINE__ << std::endl; std::exit(1); @@ -5877,6 +6097,14 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc i++; continue; } + // qwen35 GDN l2 norms are emitted as rms_norm + scalar scale (models.h + // build_gdn_l2_norm), which the rms_norm+mul fusion above cannot match + if (node->op == GGML_OP_RMS_NORM && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_SCALE }, {})) { + ggml_sycl_op_rms_norm_scale_fused(*sycl_ctx, node, cgraph->nodes[i + 1]); + i++; + continue; + } if (node->op == GGML_OP_ADD && ggml_sycl_can_fuse(cgraph, i, { GGML_OP_ADD, GGML_OP_ADD }, {})) { ggml_sycl_op_add_add_fused(*sycl_ctx, node, cgraph->nodes[i + 1]); @@ -5890,6 +6118,31 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc continue; } + // Batch consecutive independent same-shape F32 L2_NORM siblings (the GDN q/k + // norms) into one launch; sources are strided views of the fused qkv buffer, so + // the scan skips the interleaved view nodes instead of breaking on them. + if (node->op == GGML_OP_L2_NORM) { + const int l2_batch_skip = ggml_sycl_l2_norm_batch_fused(*sycl_ctx, cgraph, i); + if (l2_batch_skip > 0) { + i += l2_batch_skip; + continue; + } + } + + if (node->op == GGML_OP_SSM_CONV && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_SSM_CONV, GGML_OP_ADD, GGML_OP_UNARY }, { GGML_UNARY_OP_SILU })) { + ggml_sycl_ssm_conv_fused(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]); + i += 2; + continue; + } + + if (node->op == GGML_OP_SSM_CONV && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_SSM_CONV, GGML_OP_UNARY }, { GGML_UNARY_OP_SILU })) { + ggml_sycl_ssm_conv_fused(*sycl_ctx, node, nullptr, cgraph->nodes[i + 1]); + i++; + continue; + } + if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_mmvq_fused(*sycl_ctx, cgraph, i)) { i += 2; continue; @@ -6058,7 +6311,7 @@ bool ggml_backend_is_sycl(ggml_backend_t backend) { return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_sycl_guid()); } -int ggml_backend_sycl_get_device_count() { +int ggml_backend_sycl_get_device_count(void) { return ggml_sycl_info().device_count; } @@ -6076,12 +6329,13 @@ static const char * ggml_backend_sycl_device_get_description(ggml_backend_dev_t } static void ggml_backend_sycl_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); ggml_backend_sycl_device_context * ctx = (ggml_backend_sycl_device_context *) dev->context; - bool res = get_memory_size(dpct::dev_mgr::instance().get_device(ctx->device), *free, *total, - (MemoryAPIType) g_ggml_sycl_get_mem_api); - if (!res) { + if (!sycl_get_mem_info(ctx->device, free, total)) { GGML_ABORT("[%s] failed to get device memory size", __func__); } + GGML_SYCL_DEBUG("[SYCL] call %s total %zu free %zu\n", __func__, *total, *free); + ggml_sycl_memtrace_report_device("device memory query (dev)", ctx->device, *free, *total); } static enum ggml_backend_dev_type ggml_backend_sycl_device_get_type(ggml_backend_dev_t dev) { @@ -6127,8 +6381,8 @@ static ggml_backend_buffer_type_t ggml_backend_sycl_device_get_buffer_type(ggml_ } static ggml_backend_buffer_type_t ggml_backend_sycl_device_get_host_buffer_type(ggml_backend_dev_t dev) { - GGML_UNUSED(dev); - return ggml_backend_sycl_host_buffer_type(); + ggml_backend_sycl_device_context * ctx = (ggml_backend_sycl_device_context *) dev->context; + return ggml_backend_sycl_host_buffer_type_for_device(ctx->device); } static ggml_backend_buffer_t ggml_backend_sycl_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { @@ -6224,7 +6478,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons return false; } - if (src0_type == GGML_TYPE_TQ2_0) { + if (src0_type == GGML_TYPE_TQ2_0 || src0_type == GGML_TYPE_TQ1_0) { return false; } @@ -6270,6 +6524,12 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons return false; } } + case GGML_OP_GET_ROWS_BACK: + // return true; + return op->type == GGML_TYPE_F32 && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + op->src[1]->type == GGML_TYPE_I32 && + op->ne[2] == 1 && op->ne[3] == 1; case GGML_OP_SET: return (op->type == GGML_TYPE_F32) && (op->src[0] && op->src[1]) && @@ -6278,7 +6538,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_SET_ROWS: { - if (op->type == GGML_TYPE_TQ2_0) { + if (op->type == GGML_TYPE_TQ2_0 || op->type == GGML_TYPE_TQ1_0) { return false; } auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || @@ -6295,7 +6555,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons op->src[2]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; case GGML_OP_DSV4_HC_POST: return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && - op->src[2]->type == GGML_TYPE_F32 && op->src[3]->type == GGML_TYPE_F32 && + op->src[2]->type == GGML_TYPE_F32 && + (op->src[3] == nullptr || op->src[3]->type == GGML_TYPE_F32) && op->type == GGML_TYPE_F32; case GGML_OP_LIGHTNING_INDEXER: return op->src[0]->type == GGML_TYPE_F32 && @@ -6400,12 +6661,14 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons src1_type == GGML_TYPE_IQ3_S || src1_type == GGML_TYPE_IQ1_S || src1_type == GGML_TYPE_IQ1_M || - src1_type == GGML_TYPE_TQ2_0) { + src1_type == GGML_TYPE_TQ2_0 || + src1_type == GGML_TYPE_TQ1_0) { return false; } } - if (src0_type == GGML_TYPE_TQ2_0 || src1_type == GGML_TYPE_TQ2_0) { + if (src0_type == GGML_TYPE_TQ2_0 || src1_type == GGML_TYPE_TQ2_0 || + src0_type == GGML_TYPE_TQ1_0 || src1_type == GGML_TYPE_TQ1_0) { return false; } @@ -6501,7 +6764,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons op->type == GGML_TYPE_I32 && src0->type == GGML_TYPE_F32 && ggml_is_contiguous(src0) && - k > 0 && k <= 32; + k > 0 && k <= src0->ne[0]; } case GGML_OP_POOL_2D: case GGML_OP_POOL_1D: @@ -6957,6 +7220,7 @@ static const ggml_backend_reg_i ggml_backend_sycl_reg_interface = { // backend registry ggml_backend_reg_t ggml_backend_sycl_reg() { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_reg\n"); static ggml_backend_reg reg; static bool initialized = false; @@ -6964,7 +7228,7 @@ ggml_backend_reg_t ggml_backend_sycl_reg() { static std::mutex mutex; std::lock_guard<std::mutex> lock(mutex); if (!initialized) { - initialize_sycl_begining(); + ggml_check_sycl(); ggml_backend_sycl_reg_context * ctx = new ggml_backend_sycl_reg_context; const int min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32; diff --git a/ggml/src/ggml-sycl/mem.cpp b/ggml/src/ggml-sycl/mem.cpp index 5ec466420e02..ad5bfe0ff1eb 100644 --- a/ggml/src/ggml-sycl/mem.cpp +++ b/ggml/src/ggml-sycl/mem.cpp @@ -6,13 +6,13 @@ #include <level_zero/zes_api.h> #endif -#include "base.hpp" -#include "mem.hpp" - #include <cstdint> #include <iostream> #include <vector> +#include "base.hpp" +#include "mem.hpp" + const char * mem_api_int2str(int mem_api) { if (mem_api == MEMORY_API_TYPE_SYCL) { return "SYCL API"; @@ -24,7 +24,12 @@ const char * mem_api_int2str(int mem_api) { } #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API +/* +* Depend on to call zesInit(0) before any other Level Zero API calls, otherwise the Level Zero API calls may fail. +*/ bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & total_bytes) { + GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using Level Zero API.\n", __func__); + free_bytes = 0; total_bytes = 0; @@ -37,41 +42,28 @@ bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & tot #endif try { - ze_result_t zes_init = zesInit(0); - if (zes_init != ZE_RESULT_SUCCESS) { - std::cerr << "Warning: zesInit failed with code " << static_cast<int>(zes_init) - << ". Sysman free-memory query may be unavailable.\n"; - } if (dev.get_platform().get_backend() != kL0Backend) { - GGML_SYCL_DEBUG("Device backend is not Level Zero; falling back to SYCL memory query.\n"); - total_bytes = dev.get_info<sycl::info::device::global_mem_size>(); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("Device backend is not Level Zero.\n"); return false; } ze_device_handle_t ze_dev = sycl::get_native<kL0Backend>(dev); if (ze_dev == nullptr) { - GGML_SYCL_DEBUG("Level Zero device handle is null; falling back to SYCL memory query.\n"); - total_bytes = dev.get_info<sycl::info::device::global_mem_size>(); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("Level Zero device handle is null.\n"); return false; } ze_result_t r = zesDeviceEnumMemoryModules(ze_dev, &module_count, nullptr); if (r != ZE_RESULT_SUCCESS || module_count == 0) { - GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules. Falling back to SYCL memory query.\n"); - total_bytes = dev.get_info<sycl::info::device::global_mem_size>(); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules.\n"); return false; } std::vector<zes_mem_handle_t> modules(module_count); r = zesDeviceEnumMemoryModules(ze_dev, &module_count, modules.data()); if (r != ZE_RESULT_SUCCESS || module_count == 0) { - GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules. Falling back to SYCL memory query.\n"); - total_bytes = dev.get_info<sycl::info::device::global_mem_size>(); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules.\n"); return false; } @@ -90,73 +82,70 @@ bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & tot } if (total_bytes == 0) { - GGML_SYCL_DEBUG("Level Zero memory query returned zero total bytes. Falling back to SYCL memory query.\n"); - total_bytes = dev.get_info<sycl::info::device::global_mem_size>(); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("Level Zero memory query returned zero total bytes.\n"); return false; } - return true; + return total_bytes >= free_bytes; + } catch (const sycl::exception & e) { GGML_SYCL_DEBUG("Level Zero memory query failed: %s\n", e.what()); - total_bytes = dev.get_info<sycl::info::device::global_mem_size>(); - free_bytes = total_bytes; return false; } } #endif bool get_memory_size_by_sycl_api(sycl::device dev, size_t & free_bytes, size_t & total_bytes) { - GGML_SYCL_DEBUG("[%s]Querying free memory using SYCL API.\n", __func__); + GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using SYCL API.\n", __func__); total_bytes = dev.get_info<sycl::info::device::global_mem_size>(); #if (defined(__SYCL_COMPILER_VERSION) && __SYCL_COMPILER_VERSION >= 20221105) if (dev.has(sycl::aspect::ext_intel_free_memory)) { try { - GGML_SYCL_DEBUG("Querying free memory using SYCL aspect::ext_intel_free_memory."); + GGML_SYCL_DEBUG("Querying free memory using SYCL aspect::ext_intel_free_memory.\n"); free_bytes = dev.get_info<sycl::ext::intel::info::device::free_memory>(); return true; } catch (const sycl::exception &) { GGML_SYCL_DEBUG( - "Failed to query free memory using SYCL aspect::ext_intel_free_memory. Using total memory as free " - "memory."); - free_bytes = total_bytes; + "Failed to query free memory using SYCL aspect::ext_intel_free_memory.\n"); return false; } } else { GGML_SYCL_DEBUG( - "Device does not support SYCL aspect::ext_intel_free_memory. Using total memory as free memory."); - free_bytes = total_bytes; + "Device does not support SYCL aspect::ext_intel_free_memory.\n"); } #else - GGML_SYCL_DEBUG("SYCL Compiler version is older than 20221105. Using total memory as free memory."); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("SYCL Compiler version is older than 20221105.\n"); #endif - return true; + return false; } bool get_memory_size(sycl::device dev, size_t & free_bytes, size_t & total_bytes, MemoryAPIType api_type) { - const auto name = dev.get_info<sycl::info::device::name>(); - const auto vendor = dev.get_info<sycl::info::device::vendor>(); - const auto global_mem = dev.get_info<sycl::info::device::global_mem_size>(); - GGML_SYCL_DEBUG("[%s]GPU Name: %s\n", __func__, name.c_str()); - GGML_SYCL_DEBUG("[%s]GPU Vendor: %s\n", __func__, vendor.c_str()); - GGML_SYCL_DEBUG("[%s]GPU Global Memory: %zu bytes\n", __func__, static_cast<size_t>(global_mem)); + GGML_SYCL_DEBUG("[%s]GPU Name: %s\n", __func__, + dev.get_info<sycl::info::device::name>().c_str()); + GGML_SYCL_DEBUG("[%s]GPU Vendor: %s\n", __func__, + dev.get_info<sycl::info::device::vendor>().c_str()); if (api_type == MEMORY_API_TYPE_LEVEL_ZERO) { #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API - GGML_SYCL_DEBUG("[%s]Querying free memory using Level Zero API.\n", __func__); - if (!query_free_memory_by_ze(dev, free_bytes, total_bytes)) { - //fallback to SYCL API if Level Zero API fails - GGML_SYCL_DEBUG("[%s]Falling back to SYCL API for memory query.\n", __func__); - return get_memory_size_by_sycl_api(dev, free_bytes, total_bytes); + GGML_SYCL_DEBUG("[%s] Querying free memory using Level Zero API.\n", __func__); + if (query_free_memory_by_ze(dev, free_bytes, total_bytes)) { + return true; } - return true; -#else - GGML_SYCL_DEBUG("[%s]Level Zero API support is not enabled. Please enable it to use this feature.\n", __func__); - return false; + //fallback to SYCL API if Level Zero API fails + GGML_SYCL_DEBUG("[%s] Falling back to SYCL API for memory query.\n", __func__); #endif - } else { //MEMORY_API_TYPE_SYCL - return get_memory_size_by_sycl_api(dev, free_bytes, total_bytes); } + + //MEMORY_API_TYPE_SYCL + if(get_memory_size_by_sycl_api(dev, free_bytes, total_bytes)){ + return true; + } + + //Todo, fallback to other methods to get free memory size, such as using OS-specific APIs (e.g., /proc/meminfo on Linux, GlobalMemoryStatusEx on Windows, etc.) + GGML_SYCL_DEBUG( + "[%s] Can't get free mem size by Level Zero and SYCL API. Using total memory as free memory.\n", __func__); + free_bytes = total_bytes; + + return true; } diff --git a/ggml/src/ggml-sycl/memtrace.cpp b/ggml/src/ggml-sycl/memtrace.cpp new file mode 100644 index 000000000000..9c4f8853916d --- /dev/null +++ b/ggml/src/ggml-sycl/memtrace.cpp @@ -0,0 +1,194 @@ +#include "memtrace.hpp" + +#include "common.hpp" +#include "ggml-impl.h" + +#include <cstdio> +#include <mutex> +#include <unordered_map> + +constexpr size_t MIB = 1024 * 1024; + +static const char * mem_type_name(ggml_sycl_mem_type type) { + switch (type) { + case GGML_SYCL_MEM_BUFFER: return "buffer"; + case GGML_SYCL_MEM_POOL_LEG: return "pool_leg"; + case GGML_SYCL_MEM_POOL_VMM: return "pool_vmm"; + case GGML_SYCL_MEM_ASYNC: return "async"; + case GGML_SYCL_MEM_FATTN_KV: return "fattn_kv"; + case GGML_SYCL_MEM_DIRECT: return "direct"; + default: GGML_ABORT("[%s] The type value %d is not supported\n", __func__, (int) type); + } +} + +struct mem_tracker { + std::mutex mutex; + std::unordered_map<const void *, std::pair<ggml_sycl_mem_type, size_t>> live_by_ptr; + size_t live[GGML_SYCL_MEM_TYPE_COUNT] = {}; + size_t peak[GGML_SYCL_MEM_TYPE_COUNT] = {}; + size_t total_live = 0; + size_t total_peak = 0; + size_t last_logged_peak = 0; +}; + +static mem_tracker & get_tracker() { + static mem_tracker t; + return t; +} + +static size_t step_bytes() { + const int mib = g_ggml_sycl_memtrace_step > 0 ? g_ggml_sycl_memtrace_step : 64; + return (size_t) mib * MIB; +} + +static void report_sites_locked() { + mem_tracker & t = get_tracker(); + for (int i = 0; i < GGML_SYCL_MEM_TYPE_COUNT; i++) { + if (t.peak[i] == 0) { + continue; + } + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " %-9s allocated %5zu MiB, peak %5zu MiB\n", + mem_type_name((ggml_sycl_mem_type) i), t.live[i] / MIB, t.peak[i] / MIB); + } +} + +static void report_locked(const char * tag) { + mem_tracker & t = get_tracker(); + + const size_t allocated = t.total_live / MIB; + const size_t buffers = t.live[GGML_SYCL_MEM_BUFFER] / MIB; + + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " %s: allocated %5zu MiB (buffers %5zu + scratch %5zu)," + " peak %5zu MiB\n", + tag, allocated, buffers, allocated - buffers, t.total_peak / MIB); + report_sites_locked(); +} + +static void log_event_locked(const char * op, ggml_sycl_mem_type type, const void * ptr, size_t bytes) { + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " allocated %5zu MiB %-5s %-9s %9.3f MiB ptr=%p\n", + get_tracker().total_live / MIB, op, mem_type_name(type), + (double) bytes / MIB, ptr); +} + +bool ggml_sycl_memtrace_enabled() { + return g_ggml_sycl_memtrace > 0; +} + +void ggml_sycl_memtrace_add(ggml_sycl_mem_type type, const void * ptr, size_t bytes) { + if (!ggml_sycl_memtrace_enabled()) { + return; + } + GGML_ASSERT(ptr != nullptr); + GGML_ASSERT(bytes != 0); + + mem_tracker & t = get_tracker(); + std::lock_guard<std::mutex> lock(t.mutex); + + auto it = t.live_by_ptr.find(ptr); + if (it != t.live_by_ptr.end()) { + t.live[it->second.first] -= it->second.second; + t.total_live -= it->second.second; + } + + t.live_by_ptr[ptr] = { type, bytes }; + t.live[type] += bytes; + t.total_live += bytes; + + if (t.live[type] > t.peak[type]) { + t.peak[type] = t.live[type]; + } + if (t.total_live > t.total_peak) { + t.total_peak = t.total_live; + } + + if (g_ggml_sycl_memtrace >= 2) { + log_event_locked("alloc", type, ptr, bytes); + } + + static const size_t step = step_bytes(); + if (t.total_peak >= t.last_logged_peak + step) { + t.last_logged_peak = t.total_peak; + char tag[96]; + std::snprintf(tag, sizeof(tag), "peak grew (+%zu MiB from %s)", bytes / MIB, + mem_type_name(type)); + report_locked(tag); + } +} + +void ggml_sycl_memtrace_del(const void * ptr) { + if (!ggml_sycl_memtrace_enabled() || ptr == nullptr) { + return; + } + mem_tracker & t = get_tracker(); + std::lock_guard<std::mutex> lock(t.mutex); + + auto it = t.live_by_ptr.find(ptr); + if (it == t.live_by_ptr.end()) { + return; + } + const ggml_sycl_mem_type type = it->second.first; + const size_t bytes = it->second.second; + t.live[type] -= bytes; + t.total_live -= bytes; + t.live_by_ptr.erase(it); + + if (g_ggml_sycl_memtrace >= 2) { + log_event_locked("free", type, ptr, bytes); + } +} + +void ggml_sycl_memtrace_fail(ggml_sycl_mem_type type, size_t bytes) { + GGML_LOG_ERROR(GGML_SYCL_MEMTRACE_TAG " alloc FAILED: %9.3f MiB %s\n", + (double) bytes / MIB, mem_type_name(type)); + if (!ggml_sycl_memtrace_enabled()) { + return; + } + mem_tracker & t = get_tracker(); + std::lock_guard<std::mutex> lock(t.mutex); + report_locked("at allocation failure"); +} + +void ggml_sycl_memtrace_report(const char * tag) { + if (!ggml_sycl_memtrace_enabled()) { + return; + } + mem_tracker & t = get_tracker(); + std::lock_guard<std::mutex> lock(t.mutex); + report_locked(tag); +} + +static bool device_memory_is_dedicated(int device) { + if (device < 0 || device >= ggml_sycl_info().device_count) { + return false; + } + const sycl_device_info & info = ggml_sycl_info().devices[device]; + return info.l0_device_type_valid && info.l0_discrete_gpu; +} + +void ggml_sycl_memtrace_report_device(const char * tag, int device, size_t dev_free, size_t dev_total) { + if (!ggml_sycl_memtrace_enabled()) { + return; + } + mem_tracker & t = get_tracker(); + std::lock_guard<std::mutex> lock(t.mutex); + + const size_t in_use = dev_total > dev_free ? dev_total - dev_free : 0; + const size_t total = dev_total / MIB; + const size_t freed = dev_free / MIB; + const size_t allocated = t.total_live / MIB; + const size_t buffers = t.live[GGML_SYCL_MEM_BUFFER] / MIB; + const size_t peak = t.total_peak / MIB; + + if (in_use >= t.total_live && device_memory_is_dedicated(device) && total >= freed + allocated) { + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " %s: total %5zu MiB = free %5zu + allocated %5zu" + " (buffers %5zu + scratch %5zu) + other %5zu, peak %5zu MiB\n", + tag, total, freed, allocated, buffers, allocated - buffers, + total - freed - allocated, peak); + } else { + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " %s: total %5zu MiB, free %5zu, in use %5zu;" + " allocated %5zu (buffers %5zu + scratch %5zu), peak %5zu MiB\n", + tag, total, freed, in_use / MIB, allocated, buffers, + allocated - buffers, peak); + } + report_sites_locked(); +} diff --git a/ggml/src/ggml-sycl/memtrace.hpp b/ggml/src/ggml-sycl/memtrace.hpp new file mode 100644 index 000000000000..426d90963ab8 --- /dev/null +++ b/ggml/src/ggml-sycl/memtrace.hpp @@ -0,0 +1,28 @@ +#ifndef GGML_SYCL_MEMTRACE_HPP +#define GGML_SYCL_MEMTRACE_HPP + +#include <cstddef> + +#define GGML_SYCL_MEMTRACE_TAG "[SYCL-MEMTRACE]" + +enum ggml_sycl_mem_type { + GGML_SYCL_MEM_BUFFER = 0, + GGML_SYCL_MEM_POOL_LEG, + GGML_SYCL_MEM_POOL_VMM, + GGML_SYCL_MEM_ASYNC, + GGML_SYCL_MEM_FATTN_KV, + GGML_SYCL_MEM_DIRECT, + + GGML_SYCL_MEM_TYPE_COUNT, +}; + +bool ggml_sycl_memtrace_enabled(); + +void ggml_sycl_memtrace_add(ggml_sycl_mem_type type, const void * ptr, size_t bytes); +void ggml_sycl_memtrace_del(const void * ptr); + +void ggml_sycl_memtrace_report(const char * tag); +void ggml_sycl_memtrace_report_device(const char * tag, int device, size_t dev_free, size_t dev_total); +void ggml_sycl_memtrace_fail(ggml_sycl_mem_type type, size_t bytes); + +#endif // GGML_SYCL_MEMTRACE_HPP diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 933bc77d2e44..7e4f22dd18a8 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -2671,6 +2671,34 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens GGML_UNUSED(ctx); } +// vec_dot_q_sycl_t adapters for the IQ vec_dots that take their codebook tables as extra +// arguments: bind the constant tables here (as vec_dot_iq2_s_q8_1 / vec_dot_iq1_m_q8_1 already do +// internally) so they can be used as template arguments of mul_mat_vec_q_moe. +static __dpct_inline__ float vec_dot_iq2_xxs_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, const int & iqs) { + return vec_dot_iq2_xxs_q8_1(vbq, bq8_1, iqs, iq2xxs_grid, ksigns_iq2xs, kmask_iq2xs); +} + +static __dpct_inline__ float vec_dot_iq2_xs_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, const int & iqs) { + return vec_dot_iq2_xs_q8_1(vbq, bq8_1, iqs, iq2xs_grid, ksigns64); +} + +static __dpct_inline__ float vec_dot_iq3_xxs_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, const int & iqs) { + return vec_dot_iq3_xxs_q8_1(vbq, bq8_1, iqs, iq3xxs_grid, ksigns64); +} + +static __dpct_inline__ float vec_dot_iq3_s_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, const int & iqs) { + return vec_dot_iq3_s_q8_1(vbq, bq8_1, iqs, iq3s_grid); +} + +static __dpct_inline__ float vec_dot_iq1_s_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, const int & iqs) { + return vec_dot_iq1_s_q8_1(vbq, bq8_1, iqs, iq1s_grid_gpu); +} + // src1_row_stride: 0 for shared src1 (gate/up proj), else per-expert stride (down proj). template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl> static void mul_mat_vec_q_moe( @@ -2822,6 +2850,51 @@ bool ggml_sycl_mul_mat_vec_q_id( vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, expert_weight_stride, dst_row_stride, src1_row_stride, stream); return true; + case GGML_TYPE_IQ2_XXS: + launch_mul_mat_vec_q_moe<QK_K, QI2_XXS/2, block_iq2_xxs, VDR_IQ2_XXS_Q8_1_MMVQ, vec_dot_iq2_xxs_q8_1_moe>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; + case GGML_TYPE_IQ2_XS: + launch_mul_mat_vec_q_moe<QK_K, QI2_XS/2, block_iq2_xs, VDR_IQ2_XS_Q8_1_MMVQ, vec_dot_iq2_xs_q8_1_moe>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; + case GGML_TYPE_IQ2_S: + launch_mul_mat_vec_q_moe<QK_K, QI2_S/2, block_iq2_s, VDR_IQ2_S_Q8_1_MMVQ, vec_dot_iq2_s_q8_1>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; + case GGML_TYPE_IQ3_XXS: + launch_mul_mat_vec_q_moe<QK_K, QI3_XXS/2, block_iq3_xxs, VDR_IQ3_XXS_Q8_1_MMVQ, vec_dot_iq3_xxs_q8_1_moe>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; + case GGML_TYPE_IQ3_S: + launch_mul_mat_vec_q_moe<QK_K, QI3_S/2, block_iq3_s, VDR_IQ3_S_Q8_1_MMVQ, vec_dot_iq3_s_q8_1_moe>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; + case GGML_TYPE_IQ1_S: + launch_mul_mat_vec_q_moe<QK_K, QI1_S, block_iq1_s, VDR_IQ1_S_Q8_1_MMVQ, vec_dot_iq1_s_q8_1_moe>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; + case GGML_TYPE_IQ1_M: + launch_mul_mat_vec_q_moe<QK_K, QI1_S, block_iq1_m, VDR_IQ1_M_Q8_1_MMVQ, vec_dot_iq1_m_q8_1>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; + case GGML_TYPE_IQ4_NL: + launch_mul_mat_vec_q_moe<QK4_NL, QI4_NL, block_iq4_nl, VDR_IQ4_NL_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; + case GGML_TYPE_IQ4_XS: + launch_mul_mat_vec_q_moe<QK_K, QI4_XS/4, block_iq4_xs, VDR_IQ4_XS_Q8_1_MMVQ, vec_dot_iq4_xs_q8_1>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; default: return false; } @@ -2978,6 +3051,191 @@ static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate launch_mul_mat_vec_q_reorder_glu_impl<reorder_vec_dot_q_sycl, ncols_dst, rows_per_sg>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); } +// --------------------------------------------------------------------------- +// Fused dense-FFN GEMV + GLU over the standard (non-reorder) weight layout. +// +// Unlike the reorder variant below, the two weights may carry different block +// types (e.g. an unsloth UD mix with an iq4_xs gate and a q5_K up), as long as +// both quantize in QK_K-sized super-blocks so that one q8_1 activation +// quantization serves both dots. Per-operand accumulation order matches +// mul_mat_vec_q exactly, so results are bit-identical to running the three +// nodes separately. +// --------------------------------------------------------------------------- +template <int qi_g, typename block_g_t, int vdr_g, vec_dot_q_sycl_t vec_dot_g, + int qi_u, typename block_u_t, int vdr_u, vec_dot_q_sycl_t vec_dot_u, int ncols_dst> +static void mul_mat_vec_q_glu(const void * __restrict__ vxg, const void * __restrict__ vxu, + const void * __restrict__ vy, float * __restrict__ dst, const int ncols, + const int nrows, const int stride_col_y, const int stride_col_dst, + const ggml_glu_op glu_op, const sycl::nd_item<3> & item_ct1) { + static_assert(QK_K % QK8_1 == 0); + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); + if (row >= nrows) { + return; + } + const int blocks_per_row = ncols / QK_K; + constexpr int blocks_per_warp_g = (vdr_g * WARP_SIZE + qi_g - 1) / qi_g; + constexpr int blocks_per_warp_u = (vdr_u * WARP_SIZE + qi_u - 1) / qi_u; + // one partial sum per output column, per operand + float tmpg[ncols_dst] = {0.0f}; + float tmpu[ncols_dst] = {0.0f}; + const block_g_t * xg = (const block_g_t *) vxg; + const block_u_t * xu = (const block_u_t *) vxu; + const block_q8_1 * y = (const block_q8_1 *) vy; + for (int i = item_ct1.get_local_id(2) / (qi_g / vdr_g); i < blocks_per_row; i += blocks_per_warp_g) { + const int ibx = row * blocks_per_row + i; + const int iby = i * (QK_K / QK8_1); + for (size_t elem = 0; elem < qi_g / vdr_g; elem += WARP_SIZE) { + const int iqs = elem + vdr_g * (item_ct1.get_local_id(2) % (qi_g / vdr_g)); +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + tmpg[j] += vec_dot_g(&xg[ibx], &y[j * stride_col_y + iby], iqs); + } + } + } + for (int i = item_ct1.get_local_id(2) / (qi_u / vdr_u); i < blocks_per_row; i += blocks_per_warp_u) { + const int ibx = row * blocks_per_row + i; + const int iby = i * (QK_K / QK8_1); + for (size_t elem = 0; elem < qi_u / vdr_u; elem += WARP_SIZE) { + const int iqs = elem + vdr_u * (item_ct1.get_local_id(2) % (qi_u / vdr_u)); +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + tmpu[j] += vec_dot_u(&xu[ibx], &y[j * stride_col_y + iby], iqs); + } + } + } + // sum up partial sums and write back the activated product +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { +#pragma unroll + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { + tmpg[j] += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmpg[j], mask); + tmpu[j] += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmpu[j], mask); + } + } + if (item_ct1.get_local_id(2) == 0) { +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + // uniform across the launch; the dispatcher only accepts SWIGLU and GEGLU + const float gate = glu_op == GGML_GLU_OP_SWIGLU ? op_silu(tmpg[j]) : op_gelu(tmpg[j]); + dst[j * stride_col_dst + row] = gate * tmpu[j]; + } + } +} + +template <int qi_g, typename block_g_t, int vdr_g, vec_dot_q_sycl_t vec_dot_g, + int qi_u, typename block_u_t, int vdr_u, vec_dot_q_sycl_t vec_dot_u, int ncols_dst> +static void launch_mul_mat_vec_q_glu(const void * vxg, const void * vxu, const void * vy, float * dst, + const int ncols, const int nrows, const int stride_col_y, + const int stride_col_dst, const ggml_glu_op glu_op, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_glu<qi_g, block_g_t, vdr_g, vec_dot_g, + qi_u, block_u_t, vdr_u, vec_dot_u, ncols_dst>( + vxg, vxu, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, + glu_op, nd_item); + }); + }); +} + +// Dispatch the plain-layout fused GLU GEMV over the activation batch: ncols_dst +// selects the kernel's per-column template parameter. Returns false when the +// batch exceeds the instantiated range; the caller falls back to unfused nodes. +template <int qi_g, typename block_g_t, int vdr_g, vec_dot_q_sycl_t vec_dot_g, + int qi_u, typename block_u_t, int vdr_u, vec_dot_q_sycl_t vec_dot_u> +static bool dispatch_mul_mat_vec_q_glu_plain(const void * vgate, const void * vup, const void * vy, + float * dst, const int ncols, const int nrows, + const int stride_col_y, const int stride_col_dst, + const ggml_glu_op glu_op, dpct::queue_ptr stream, + const int ncols_dst) { + switch (ncols_dst) { + case 1: + launch_mul_mat_vec_q_glu<qi_g, block_g_t, vdr_g, vec_dot_g, + qi_u, block_u_t, vdr_u, vec_dot_u, 1>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream); + return true; + case 2: + launch_mul_mat_vec_q_glu<qi_g, block_g_t, vdr_g, vec_dot_g, + qi_u, block_u_t, vdr_u, vec_dot_u, 2>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream); + return true; + case 3: + launch_mul_mat_vec_q_glu<qi_g, block_g_t, vdr_g, vec_dot_g, + qi_u, block_u_t, vdr_u, vec_dot_u, 3>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream); + return true; + case 4: + launch_mul_mat_vec_q_glu<qi_g, block_g_t, vdr_g, vec_dot_g, + qi_u, block_u_t, vdr_u, vec_dot_u, 4>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream); + return true; + case 5: + launch_mul_mat_vec_q_glu<qi_g, block_g_t, vdr_g, vec_dot_g, + qi_u, block_u_t, vdr_u, vec_dot_u, 5>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream); + return true; + case 6: + launch_mul_mat_vec_q_glu<qi_g, block_g_t, vdr_g, vec_dot_g, + qi_u, block_u_t, vdr_u, vec_dot_u, 6>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream); + return true; + case 7: + launch_mul_mat_vec_q_glu<qi_g, block_g_t, vdr_g, vec_dot_g, + qi_u, block_u_t, vdr_u, vec_dot_u, 7>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream); + return true; + case 8: + launch_mul_mat_vec_q_glu<qi_g, block_g_t, vdr_g, vec_dot_g, + qi_u, block_u_t, vdr_u, vec_dot_u, 8>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream); + return true; + default: + return false; + } +} + +// Fused dense-FFN GEMV + GLU for weight pairs the reorder kernel does not cover. +// vgate/vup must be in the standard block layout; vy must be quantized with plain +// quantize_q8_1 (padded rows). stride_col_y is in block_q8_1 units. +// Returns false if the type pair or batch is unhandled; caller should fall back. +bool ggml_sycl_mul_mat_vec_q_glu_plain(enum ggml_type gate_type, enum ggml_type up_type, + enum ggml_glu_op glu_op, const void * vgate, const void * vup, + const void * vy, float * dst, int ncols, int nrows, int ncols_dst, + int stride_col_y, int stride_col_dst, dpct::queue_ptr stream) { + if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) { + return false; + } + if (ncols % QK_K != 0) { + return false; + } + if (gate_type == GGML_TYPE_Q5_K && up_type == GGML_TYPE_Q5_K) { + return dispatch_mul_mat_vec_q_glu_plain<QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1, + QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream, ncols_dst); + } + if (gate_type == GGML_TYPE_IQ4_XS && up_type == GGML_TYPE_IQ4_XS) { + return dispatch_mul_mat_vec_q_glu_plain<QI4_XS / 4, block_iq4_xs, VDR_IQ4_XS_Q8_1_MMVQ, vec_dot_iq4_xs_q8_1, + QI4_XS / 4, block_iq4_xs, VDR_IQ4_XS_Q8_1_MMVQ, vec_dot_iq4_xs_q8_1>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream, ncols_dst); + } + if (gate_type == GGML_TYPE_IQ4_XS && up_type == GGML_TYPE_Q5_K) { + return dispatch_mul_mat_vec_q_glu_plain<QI4_XS / 4, block_iq4_xs, VDR_IQ4_XS_Q8_1_MMVQ, vec_dot_iq4_xs_q8_1, + QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream, ncols_dst); + } + if (gate_type == GGML_TYPE_Q5_K && up_type == GGML_TYPE_IQ4_XS) { + return dispatch_mul_mat_vec_q_glu_plain<QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1, + QI4_XS / 4, block_iq4_xs, VDR_IQ4_XS_Q8_1_MMVQ, vec_dot_iq4_xs_q8_1>( + vgate, vup, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, glu_op, stream, ncols_dst); + } + return false; +} + bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx, const void * vgate, const void * vy, float * dst, int ncols, int nrows, int ncols_dst, int stride_col_y_bytes, int stride_col_dst, diff --git a/ggml/src/ggml-sycl/mmvq.hpp b/ggml/src/ggml-sycl/mmvq.hpp index 9d2f5645ecf3..7fb9cf6f82e6 100644 --- a/ggml/src/ggml-sycl/mmvq.hpp +++ b/ggml/src/ggml-sycl/mmvq.hpp @@ -73,4 +73,24 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder( int stride_col_dst, // floats between output columns in dst dpct::queue_ptr stream); + +// Fused dense-FFN GEMV + GLU over the standard (non-reorder) layout; the gate and up +// weights may carry different block types (q5_K / iq4_xs, mixed included). +// vy: src1 quantized with plain quantize_q8_1 (padded rows). stride_col_y is in +// block_q8_1 units. Returns false if the pair or batch is unhandled; caller falls back. +bool ggml_sycl_mul_mat_vec_q_glu_plain( + enum ggml_type gate_type, + enum ggml_type up_type, + enum ggml_glu_op glu_op, + const void * vgate, + const void * vup, + const void * vy, + float * dst, + int ncols, // K, shared by both weights + int nrows, // output rows, i.e. weight ne[1] + int ncols_dst, // activation columns, 1..MMVQ_MAX_BATCH_SIZE + int stride_col_y, // block_q8_1 units between activation columns + int stride_col_dst, // floats between output columns in dst + dpct::queue_ptr stream); + #endif // GGML_SYCL_MMVQ_HPP diff --git a/ggml/src/ggml-sycl/norm.cpp b/ggml/src/ggml-sycl/norm.cpp index 2d303372934d..36576e9c28c6 100644 --- a/ggml/src/ggml-sycl/norm.cpp +++ b/ggml/src/ggml-sycl/norm.cpp @@ -152,7 +152,8 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols, const float* mul = nullptr, const int64_t mul_stride_row = 0, const int64_t mul_stride_channel = 0, const int64_t mul_stride_sample = 0, const int mul_nrows = 0, const int mul_nchannels = 0, const int mul_nsamples = 0, const float* add = nullptr, const int64_t add_stride_row = 0, const int64_t add_stride_channel = 0, - const int64_t add_stride_sample = 0, const int add_nrows = 0, const int add_nchannels = 0, const int add_nsamples = 0) { + const int64_t add_stride_sample = 0, const int add_nrows = 0, const int add_nchannels = 0, const int add_nsamples = 0, + const float scale_mul = 1.0f) { static_assert(!do_add || do_multiply, "fusing add is not supported without multiplying"); @@ -221,7 +222,11 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols, } else if constexpr (do_multiply) { dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col]; } else { - dst[col * dst_stride_col] = scale * x[col * src_stride_col]; + // folded epilogue of a fused GGML_OP_SCALE consumer (qwen35 GDN l2 norms); + // the explicit temporary keeps the float evaluation order identical to + // running rms_norm and scale as two separate kernels + const float v = scale * x[col * src_stride_col]; + dst[col * dst_stride_col] = v * scale_mul; } } } @@ -394,6 +399,52 @@ static void rms_norm_f32_sycl(const float* x, float* dst, const int ncols, const } } +static void rms_norm_scale_f32_sycl(const float* x, float* dst, const int ncols, const int nrows, + const int nchannels, const int nsamples, + const int64_t src_stride_col, const int64_t src_stride_row, const int64_t src_stride_channel, const int64_t src_stride_sample, + const int64_t dst_stride_col, const int64_t dst_stride_row, const int64_t dst_stride_channel, const int64_t dst_stride_sample, + const float eps, const float scale_mul, queue_ptr stream, int device) { + const sycl::range<3> global_dims(nsamples, nchannels, nrows); + if (ncols < 1024) { + const sycl::range<3> block_dims(1, 1, WARP_SIZE); + stream->submit([&](sycl::handler& cgh) { + cgh.parallel_for( + sycl::nd_range<3>(global_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + rms_norm_f32(x, dst, ncols, + src_stride_col, src_stride_row, src_stride_channel, src_stride_sample, + dst_stride_col, dst_stride_row, dst_stride_channel, dst_stride_sample, + eps, item_ct1, nullptr, WARP_SIZE, + nullptr, 0, 0, 0, 0, 0, 0, + nullptr, 0, 0, 0, 0, 0, 0, + scale_mul); + }); + }); + } + else { + const int work_group_size = ggml_sycl_info().max_work_group_sizes[device]; + assert(work_group_size % (WARP_SIZE * WARP_SIZE) == 0); + const sycl::range<3> block_dims(1, 1, work_group_size); + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor<float, 1> s_sum_acc_ct1(sycl::range<1>(work_group_size / WARP_SIZE), + cgh); + cgh.parallel_for( + sycl::nd_range<3>(global_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + rms_norm_f32(x, dst, ncols, + src_stride_col, src_stride_row, src_stride_channel, src_stride_sample, + dst_stride_col, dst_stride_row, dst_stride_channel, dst_stride_sample, + eps, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size, + nullptr, 0, 0, 0, 0, 0, 0, + nullptr, 0, 0, 0, 0, 0, 0, + scale_mul); + }); + }); + } +} + static void rms_norm_mul_f32_sycl(const float* x, const float* mul, float* dst, const int ncols, const int nrows, const int nchannels, const int nsamples, const int64_t src_stride_col, const int64_t src_stride_row, const int64_t src_stride_channel, const int64_t src_stride_sample, @@ -543,6 +594,62 @@ static void l2_norm_f32_sycl(const float * x, } } +// Batched L2 norm: N independent same-shape F32 tensors in one launch; the tensor +// index is folded into grid dim0 and each row's reduction is identical to the +// single-tensor kernel, so the result is bit-exact. +struct l2_batch_ptrs { + const float * src[GGML_SYCL_L2_BATCH_MAX]; + float * dst[GGML_SYCL_L2_BATCH_MAX]; +}; + +// One stride set shared by the whole batch: the caller only groups tensors whose nb[] +// all match, so per-tensor state stays two pointers. +struct l2_batch_strides { + int ne1, ne2; + int64_t ss0, ss1, ss2, ss3; + int64_t ds0, ds1, ds2, ds3; +}; + +template <int warp_size> +static void l2_norm_f32_batch(l2_batch_ptrs p, l2_batch_strides st, const int ncols, const float eps, + const sycl::nd_item<3> & item_ct1) { + const int t = item_ct1.get_group(0); // tensor index + const int r = item_ct1.get_group(2); // flattened row over ne1*ne2*ne3 + const int tid = item_ct1.get_local_id(2); + + const int i1 = r % st.ne1; + const int i2 = (r / st.ne1) % st.ne2; + const int i3 = r / (st.ne1 * st.ne2); + + const float * x = p.src[t] + i3 * st.ss3 + i2 * st.ss2 + i1 * st.ss1; + float * dst = p.dst[t] + i3 * st.ds3 + i2 * st.ds2 + i1 * st.ds1; + + float tmp = 0.0f; + for (int col = tid; col < ncols; col += warp_size) { + const float xi = x[col * st.ss0]; + tmp += xi * xi; + } + tmp = block_reduce<block_reduce_method::SUM, warp_size>(tmp, (float *) nullptr, warp_size); + const float scale = sycl::rsqrt(sycl::fmax(tmp, eps * eps)); + for (int col = tid; col < ncols; col += warp_size) { + dst[col * st.ds0] = scale * x[col * st.ss0]; + } +} + +template <int warp_size> +static void l2_norm_f32_batch_sycl(l2_batch_ptrs p, l2_batch_strides st, const int n_tensors, + const int ncols, const int nrows_total, const float eps, + queue_ptr stream) { + const dpct::dim3 blocks_num(nrows_total, 1, n_tensors); + const dpct::dim3 block_dims(warp_size, 1, 1); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(blocks_num * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(warp_size)]] { + l2_norm_f32_batch<warp_size>(p, st, ncols, eps, item_ct1); + }); + }); +} + void ggml_sycl_op_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { const ggml_tensor * src0 = dst->src[0]; @@ -626,6 +733,46 @@ void ggml_sycl_op_rms_norm(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ss0, ss1, ss2, ss3, ds0, ds1, ds2, ds3, eps, main_stream, ctx.device); } +// Fused rms_norm + scale (the qwen35 GDN l2-norm pair build_gdn_l2_norm emits): +// the scale factor is a host scalar in the GGML_OP_SCALE node's op_params, so +// unlike the mul variants there is no second device tensor to wire up. +void ggml_sycl_op_rms_norm_scale_fused(ggml_backend_sycl_context & ctx, ggml_tensor * dst, + ggml_tensor * scale_tensor) { + const ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(scale_tensor->type == GGML_TYPE_F32); + + dpct::queue_ptr main_stream = ctx.stream(); + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + const float * src0_dd = static_cast<const float *>(src0->data); + float * dst_dd = static_cast<float *>(scale_tensor->data); + + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + float scale_mul; + memcpy(&scale_mul, scale_tensor->op_params, sizeof(float)); + GGML_ASSERT(scale_mul >= 0.0f); + + GGML_TENSOR_UNARY_OP_LOCALS + const size_t ts0 = ggml_type_size(src0->type); + const size_t tdst = ggml_type_size(scale_tensor->type); + GGML_ASSERT(nb00 % ts0 == 0 && nb01 % ts0 == 0 && nb02 % ts0 == 0 && nb03 % ts0 == 0); + GGML_ASSERT(scale_tensor->nb[0] % tdst == 0 && scale_tensor->nb[1] % tdst == 0 && + scale_tensor->nb[2] % tdst == 0 && scale_tensor->nb[3] % tdst == 0); + const int64_t ss0 = nb00 / ts0; + const int64_t ss1 = nb01 / ts0; + const int64_t ss2 = nb02 / ts0; + const int64_t ss3 = nb03 / ts0; + const int64_t ds0 = scale_tensor->nb[0] / tdst; + const int64_t ds1 = scale_tensor->nb[1] / tdst; + const int64_t ds2 = scale_tensor->nb[2] / tdst; + const int64_t ds3 = scale_tensor->nb[3] / tdst; + rms_norm_scale_f32_sycl(src0_dd, dst_dd, ne00, ne01, ne02, ne03, + ss0, ss1, ss2, ss3, ds0, ds1, ds2, ds3, eps, scale_mul, main_stream, ctx.device); +} + void ggml_sycl_op_rms_norm_fused(ggml_backend_sycl_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor) { const ggml_tensor * rms_norm_src = dst->src[0]; float eps = 0.0f; @@ -961,3 +1108,30 @@ void ggml_sycl_op_l2_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { l2_norm_f32_sycl<WARP_SIZE>(src0_d, dst_d, ne00, ne01, ne02, ne03, ss0, ss1, ss2, ss3, ds0, ds1, ds2, ds3, eps, stream, ctx.device); } + +// nodes[0..count) are independent, same-shape, same-eps, same-nb L2_NORM ops validated +// by the caller; requires ncols < 1024 (the warp reduction path). +void ggml_sycl_l2_norm_batch(ggml_backend_sycl_context & ctx, ggml_tensor ** nodes, int count) { + const ggml_tensor * s0 = nodes[0]->src[0]; + const int ncols = (int) s0->ne[0]; + const int nrows_total = (int) ggml_nrows(s0); + float eps; + memcpy(&eps, nodes[0]->op_params, sizeof(float)); + GGML_ASSERT(eps >= 0.0f); + + l2_batch_ptrs p{}; + for (int t = 0; t < count; ++t) { + p.src[t] = (const float *) nodes[t]->src[0]->data; + p.dst[t] = (float *) nodes[t]->data; + } + + const ggml_tensor * d0 = nodes[0]; + const size_t ts = ggml_type_size(GGML_TYPE_F32); + l2_batch_strides st{}; + st.ne1 = (int) s0->ne[1]; + st.ne2 = (int) s0->ne[2]; + st.ss0 = s0->nb[0] / ts; st.ss1 = s0->nb[1] / ts; st.ss2 = s0->nb[2] / ts; st.ss3 = s0->nb[3] / ts; + st.ds0 = d0->nb[0] / ts; st.ds1 = d0->nb[1] / ts; st.ds2 = d0->nb[2] / ts; st.ds3 = d0->nb[3] / ts; + + l2_norm_f32_batch_sycl<WARP_SIZE>(p, st, count, ncols, nrows_total, eps, ctx.stream()); +} diff --git a/ggml/src/ggml-sycl/norm.hpp b/ggml/src/ggml-sycl/norm.hpp index ef7b2d386bdd..fb667ac6b6e2 100644 --- a/ggml/src/ggml-sycl/norm.hpp +++ b/ggml/src/ggml-sycl/norm.hpp @@ -21,6 +21,8 @@ void ggml_sycl_op_rms_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst); void ggml_sycl_op_rms_norm_fused(ggml_backend_sycl_context& ctx, ggml_tensor* dst, ggml_tensor* mul); +void ggml_sycl_op_rms_norm_scale_fused(ggml_backend_sycl_context& ctx, ggml_tensor* dst, ggml_tensor* scale_tensor); + void ggml_sycl_op_rms_norm_fused_add(ggml_backend_sycl_context& ctx, ggml_tensor* dst, ggml_tensor* mul_tensor, ggml_tensor* add_tensor); void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context& ctx, ggml_tensor* dst); @@ -29,4 +31,7 @@ void ggml_sycl_op_group_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst); void ggml_sycl_op_l2_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst); +#define GGML_SYCL_L2_BATCH_MAX 8 +void ggml_sycl_l2_norm_batch(ggml_backend_sycl_context & ctx, ggml_tensor ** nodes, int count); + #endif // GGML_SYCL_NORM_HPP diff --git a/ggml/src/ggml-sycl/ssm_conv.cpp b/ggml/src/ggml-sycl/ssm_conv.cpp index 3eafa1a680d3..a8714351866f 100644 --- a/ggml/src/ggml-sycl/ssm_conv.cpp +++ b/ggml/src/ggml-sycl/ssm_conv.cpp @@ -1,11 +1,71 @@ #include "ssm_conv.hpp" #include "common.hpp" +#include "element_wise.hpp" #include <cstdio> using namespace sycl; -static void kernel_ssm_conv( +// One output element of the conv. DC is d_conv as a compile-time constant (0 keeps the +// runtime loop); unfused callers pass literal false/nullptr so the epilogue folds away. +template <int DC> +static __dpct_inline__ void ssm_conv_element( + size_t idx, + const float *src_data, + const float *weights, + float *dst_data, + int d_conv, + int d_inner, + int n_t, + int src_stride_inner, + int src_stride_seq, + int dst_stride_token, + int dst_stride_seq, + bool apply_silu, + const float *bias +) { + // src is token-contiguous per channel, dst is channel-contiguous per token, + // so indexing token-fastest coalesces the d_conv loads. + const int token = static_cast<int>(idx % n_t); + const int channel = static_cast<int>((idx / n_t) % d_inner); + const int seq = static_cast<int>(idx / (static_cast<size_t>(n_t) * static_cast<size_t>(d_inner))); + + const float *s = src_data + + static_cast<size_t>(seq) * static_cast<size_t>(src_stride_seq) + + static_cast<size_t>(channel) * static_cast<size_t>(src_stride_inner) + + static_cast<size_t>(token); + + const float *c = weights + static_cast<size_t>(channel) * static_cast<size_t>(d_conv); + + float sumf = 0.0f; + if constexpr (DC > 0) { +#pragma unroll + for (int i0 = 0; i0 < DC; ++i0) { + sumf += s[i0] * c[i0]; + } + } else { + for (int i0 = 0; i0 < d_conv; ++i0) { + sumf += s[i0] * c[i0]; + } + } + + // fused bias add: the ADD node broadcasts a 1-D channel bias over tokens + if (bias != nullptr) { + sumf += bias[channel]; + } + + const size_t dst_idx = + static_cast<size_t>(seq) * static_cast<size_t>(dst_stride_seq) + + static_cast<size_t>(token) * static_cast<size_t>(dst_stride_token) + + static_cast<size_t>(channel); + + dst_data[dst_idx] = apply_silu ? op_silu(sumf) : sumf; +} + +// FUSED=false keeps apply_silu/bias out of the kernel capture list, so the unfused launch +// takes the pre-fusion argument list; matters at n_t == 1, where the op is launch-bound. +template <int DC, bool FUSED> +static void kernel_ssm_conv_impl( queue &q, const float *src_data, const float *weights, @@ -18,7 +78,9 @@ static void kernel_ssm_conv( int src_stride_inner, int src_stride_seq, int dst_stride_token, - int dst_stride_seq + int dst_stride_seq, + bool apply_silu, + const float *bias ) { const size_t total_work = static_cast<size_t>(d_inner) * static_cast<size_t>(n_t) * static_cast<size_t>(n_s); const size_t work_group_size = 256; @@ -27,53 +89,199 @@ static void kernel_ssm_conv( const range<1> global_range(num_work_groups * work_group_size); const range<1> local_range(work_group_size); - q.submit([&](handler &h) { - h.parallel_for( - nd_range<1>(global_range, local_range), - [=](nd_item<1> item) { - const size_t idx = item.get_global_id(0); - if (idx >= total_work) { - return; + if constexpr (FUSED) { + q.submit([&](handler &h) { + h.parallel_for( + nd_range<1>(global_range, local_range), + [=](nd_item<1> item) { + const size_t idx = item.get_global_id(0); + if (idx >= total_work) { + return; + } + + ssm_conv_element<DC>(idx, src_data, weights, dst_data, d_conv, d_inner, n_t, + src_stride_inner, src_stride_seq, dst_stride_token, + dst_stride_seq, apply_silu, bias); } + ); + }); + } else { + GGML_UNUSED(apply_silu); + GGML_UNUSED(bias); - // src has the tokens of one channel contiguous, dst has the channels of one - // token contiguous, so either the loads or the store must be strided. Indexing - // token-fastest coalesces the d_conv loads, which measured faster except for - // short, cache-resident rows. - const int token = static_cast<int>(idx % n_t); - const int channel = static_cast<int>((idx / n_t) % d_inner); - const int seq = static_cast<int>(idx / (static_cast<size_t>(n_t) * static_cast<size_t>(d_inner))); + q.submit([&](handler &h) { + h.parallel_for( + nd_range<1>(global_range, local_range), + [=](nd_item<1> item) { + const size_t idx = item.get_global_id(0); + if (idx >= total_work) { + return; + } - const float *s = src_data - + static_cast<size_t>(seq) * static_cast<size_t>(src_stride_seq) - + static_cast<size_t>(channel) * static_cast<size_t>(src_stride_inner) - + static_cast<size_t>(token); + ssm_conv_element<DC>(idx, src_data, weights, dst_data, d_conv, d_inner, n_t, + src_stride_inner, src_stride_seq, dst_stride_token, + dst_stride_seq, false, nullptr); + } + ); + }); + } +} - const float *c = weights + static_cast<size_t>(channel) * static_cast<size_t>(d_conv); +// SLM transpose tile: coalesces both the loads and the stores. The +1 pad makes the row +// stride 33, coprime with 32 banks, so both phases are bank-conflict-free. +template <int DC, int TT, int TC, int WG> +static __dpct_inline__ void ssm_conv_tile( + nd_item<1> it, local_accessor<float, 1> tile, const float *src_data, const float *weights, + float *dst_data, int n_t, int nt_tiles, int nc_tiles, int src_stride_inner, + int src_stride_seq, int dst_stride_token, int dst_stride_seq, bool apply_silu, + const float *bias +) { + const int lid = static_cast<int>(it.get_local_id(0)); + const size_t g = it.get_group(0); + const int tt = static_cast<int>(g % nt_tiles); + const int ct = static_cast<int>((g / nt_tiles) % nc_tiles); + const int seq = static_cast<int>(g / (static_cast<size_t>(nt_tiles) * nc_tiles)); + const int t0 = tt * TT, c0 = ct * TC; - float sumf = 0.0f; - for (int i0 = 0; i0 < d_conv; ++i0) { - sumf += s[i0] * c[i0]; - } + const int ti = lid % TT; + const int cj = lid / TT; +#pragma unroll + for (int r = 0; r < TC / (WG / TT); ++r) { + const int c = cj + r * (WG / TT); + const int tok = t0 + ti; + float sumf = 0.0f; + if (tok < n_t) { + const float *s = src_data + static_cast<size_t>(seq) * src_stride_seq + + static_cast<size_t>(c0 + c) * src_stride_inner + tok; + const float *cw = weights + static_cast<size_t>(c0 + c) * DC; +#pragma unroll + for (int i = 0; i < DC; ++i) sumf += s[i] * cw[i]; + if (bias != nullptr) sumf += bias[c0 + c]; + if (apply_silu) sumf = op_silu(sumf); + } + tile[c * (TT + 1) + ti] = sumf; + } + it.barrier(access::fence_space::local_space); + + const int cc = lid % TC; + const int tj = lid / TC; +#pragma unroll + for (int r = 0; r < TT / (WG / TC); ++r) { + const int t = tj + r * (WG / TC); + const int tok = t0 + t; + if (tok < n_t) { + dst_data[static_cast<size_t>(seq) * dst_stride_seq + + static_cast<size_t>(tok) * dst_stride_token + c0 + cc] + = tile[cc * (TT + 1) + t]; + } + } +} + +// Same FUSED split as kernel_ssm_conv_impl. The fused instantiation keeps the runtime +// apply_silu/bias branches: at n_t >= 32 they are amortized over the whole tile. +template <int DC, bool FUSED> +static void kernel_ssm_conv_tiled( + queue &q, const float *src_data, const float *weights, float *dst_data, + int d_inner, int n_t, int n_s, int src_stride_inner, int src_stride_seq, + int dst_stride_token, int dst_stride_seq, bool apply_silu, const float *bias +) { + constexpr int TT = 32, TC = 32, WG = 256; + const int nt_tiles = (n_t + TT - 1) / TT; + const int nc_tiles = d_inner / TC; + const size_t groups = static_cast<size_t>(nt_tiles) * nc_tiles * n_s; - const size_t dst_idx = - static_cast<size_t>(seq) * static_cast<size_t>(dst_stride_seq) + - static_cast<size_t>(token) * static_cast<size_t>(dst_stride_token) + - static_cast<size_t>(channel); + if constexpr (FUSED) { + q.submit([&](handler &h) { + local_accessor<float, 1> tile(range<1>(TC * (TT + 1)), h); + h.parallel_for(nd_range<1>(range<1>(groups * WG), range<1>(WG)), [=](nd_item<1> it) { + ssm_conv_tile<DC, TT, TC, WG>(it, tile, src_data, weights, dst_data, n_t, nt_tiles, + nc_tiles, src_stride_inner, src_stride_seq, + dst_stride_token, dst_stride_seq, apply_silu, bias); + }); + }); + } else { + GGML_UNUSED(apply_silu); + GGML_UNUSED(bias); - dst_data[dst_idx] = sumf; - } - ); - }); + q.submit([&](handler &h) { + local_accessor<float, 1> tile(range<1>(TC * (TT + 1)), h); + h.parallel_for(nd_range<1>(range<1>(groups * WG), range<1>(WG)), [=](nd_item<1> it) { + ssm_conv_tile<DC, TT, TC, WG>(it, tile, src_data, weights, dst_data, n_t, nt_tiles, + nc_tiles, src_stride_inner, src_stride_seq, + dst_stride_token, dst_stride_seq, false, nullptr); + }); + }); + } +} + +static void kernel_ssm_conv( + queue &q, + const float *src_data, + const float *weights, + float *dst_data, + int d_conv, + int d_inner, + int n_t, + int n_s, + int ncs, + int src_stride_inner, + int src_stride_seq, + int dst_stride_token, + int dst_stride_seq, + bool apply_silu, + const float *bias +) { + // Only the fused instantiations carry apply_silu/bias as kernel arguments; the plain + // ssm_conv launch keeps the argument list it had before the fusion landed. + const bool fused = apply_silu || bias != nullptr; + + // d_inner must be a multiple of 32 so the channel tiles are exact; the transpose is only + // worth it for n_t >= 32. d_conv == 4 is the only window with a DC-specialized kernel. + if (d_conv == 4 && n_t >= 32 && (d_inner % 32) == 0) { + if (fused) { + kernel_ssm_conv_tiled<4, true>(q, src_data, weights, dst_data, d_inner, n_t, n_s, + src_stride_inner, src_stride_seq, dst_stride_token, + dst_stride_seq, apply_silu, bias); + } else { + kernel_ssm_conv_tiled<4, false>(q, src_data, weights, dst_data, d_inner, n_t, n_s, + src_stride_inner, src_stride_seq, dst_stride_token, + dst_stride_seq, apply_silu, bias); + } + return; + } + + if (d_conv == 4) { + if (fused) { + kernel_ssm_conv_impl<4, true>(q, src_data, weights, dst_data, d_conv, d_inner, n_t, n_s, + ncs, src_stride_inner, src_stride_seq, dst_stride_token, + dst_stride_seq, apply_silu, bias); + } else { + kernel_ssm_conv_impl<4, false>(q, src_data, weights, dst_data, d_conv, d_inner, n_t, n_s, + ncs, src_stride_inner, src_stride_seq, dst_stride_token, + dst_stride_seq, apply_silu, bias); + } + return; + } + + if (fused) { + kernel_ssm_conv_impl<0, true>(q, src_data, weights, dst_data, d_conv, d_inner, n_t, n_s, + ncs, src_stride_inner, src_stride_seq, dst_stride_token, + dst_stride_seq, apply_silu, bias); + } else { + kernel_ssm_conv_impl<0, false>(q, src_data, weights, dst_data, d_conv, d_inner, n_t, n_s, + ncs, src_stride_inner, src_stride_seq, dst_stride_token, + dst_stride_seq, apply_silu, bias); + } } -inline void ggml_sycl_op_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { +inline void ggml_sycl_op_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst, ggml_tensor * silu_dst = nullptr, const float * bias = nullptr) { ggml_tensor * src0 = dst->src[0]; ggml_tensor * src1 = dst->src[1]; GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(bias == nullptr || silu_dst != nullptr); const int d_conv = src1->ne[0]; const int ncs = src0->ne[0]; @@ -104,7 +312,8 @@ inline void ggml_sycl_op_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * const float *src_data = static_cast<const float *>(src0->data); const float *weights = static_cast<const float *>(src1->data); - float *dst_data = static_cast<float *>(dst->data); + const bool apply_silu = silu_dst != nullptr; + float *dst_data = static_cast<float *>((silu_dst ? silu_dst : dst)->data); GGML_ASSERT(src_data && weights && dst_data); @@ -121,7 +330,9 @@ inline void ggml_sycl_op_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * src_stride_inner, src_stride_seq, dst_stride_token, - dst_stride_seq + dst_stride_seq, + apply_silu, + bias ); } catch (const std::exception &e) { @@ -134,3 +345,17 @@ void ggml_sycl_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); ggml_sycl_op_ssm_conv(ctx, dst); } + +// Fused ssm_conv + ADD + SiLU: write silu(conv(x) + b) straight into silu_dst, eliding the +// standalone SiLU launch and its HBM round-trip of the conv output. +void ggml_sycl_ssm_conv_fused(ggml_backend_sycl_context & ctx, ggml_tensor * dst, ggml_tensor * add, ggml_tensor * silu_dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + GGML_ASSERT(silu_dst && ggml_are_same_shape(dst, silu_dst) && silu_dst->type == GGML_TYPE_F32); + // the fused kernel reads only the ADD's bias operand; the ADD result is never written + const float * bias = nullptr; + if (add != nullptr) { + const ggml_tensor * bias_t = (add->src[0] == dst) ? add->src[1] : add->src[0]; + bias = static_cast<const float *>(bias_t->data); + } + ggml_sycl_op_ssm_conv(ctx, dst, silu_dst, bias); +} diff --git a/ggml/src/ggml-sycl/ssm_conv.hpp b/ggml/src/ggml-sycl/ssm_conv.hpp index 1a8ad05f0c7f..72c9066232ef 100644 --- a/ggml/src/ggml-sycl/ssm_conv.hpp +++ b/ggml/src/ggml-sycl/ssm_conv.hpp @@ -3,3 +3,4 @@ #include "common.hpp" void ggml_sycl_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst); +void ggml_sycl_ssm_conv_fused(ggml_backend_sycl_context & ctx, ggml_tensor * dst, ggml_tensor * add, ggml_tensor * silu_dst); diff --git a/ggml/src/ggml-sycl/topk-radix.cpp b/ggml/src/ggml-sycl/topk-radix.cpp new file mode 100644 index 000000000000..8cd0bd2f5fcd --- /dev/null +++ b/ggml/src/ggml-sycl/topk-radix.cpp @@ -0,0 +1,531 @@ +#include "topk-radix.hpp" + +#include "common.hpp" + +#include <algorithm> + +// Large-k top-k by radix select on an order-preserving unsigned key. +// +// The k-th largest key of a row is found by four most-significant-first passes over its +// 8-bit digits: histogram the digit over the candidate set, walk the buckets from the +// top, and recurse into the bucket where the running count reaches what is still +// needed. Everything strictly above that bucket is in the top-k. A final pass emits +// every column whose key beats the pivot, then exactly as many pivot-equal columns as +// are still missing, so duplicate keys yield exactly k distinct indices. +// +// SLM holds only the histogram, so unlike the scan-merge kernels the cost does not grow +// with k. One work-group owns a row and runs every pass, so a top-k is one launch and +// needs no pool scratch. The row is re-read once per pass rather than compacted, which +// keeps the candidate set implicit: (key & mask) == prefix. +// +// The output is the set of winning indices in no particular order, which is what the +// reference op provides (it swaps its first two outputs to say so) and what +// test-backend-ops compares. + +static constexpr int SYCL_TOP_K_RADIX_BITS = 8; +static constexpr int SYCL_TOP_K_RADIX_BUCKETS = 1 << SYCL_TOP_K_RADIX_BITS; +// Private histogram copies, interleaved per bucket so neighbouring lanes hit +// neighbouring banks. Lanes of one instruction spread over the copies, which is what +// bounds the atomic serialisation on tie-heavy rows. +static constexpr int SYCL_TOP_K_RADIX_HIST_COPIES = 8; +static constexpr int SYCL_TOP_K_RADIX_HIST_SIZE = SYCL_TOP_K_RADIX_BUCKETS * SYCL_TOP_K_RADIX_HIST_COPIES; +// Past the histogram: pivot digit, pivot bucket count, remaining need, then the two +// emit counters. +static constexpr int SYCL_TOP_K_RADIX_SLM_WORDS = SYCL_TOP_K_RADIX_HIST_SIZE + 5; + +// Larger float <=> larger key. The reference comparator is a plain float '>', under which +// -0.0 and +0.0 tie, so -0.0 is folded onto +0.0 first. NaN has no defined order in the +// reference (its comparator is not a strict weak order on NaN); here a positive NaN keys +// above +inf and a negative NaN below -inf, which at least makes the result deterministic. +static inline uint32_t top_k_radix_key(float f) { + uint32_t u = sycl::bit_cast<uint32_t>(f); + if (u == 0x80000000u) { + u = 0u; + } + return (u & 0x80000000u) ? ~u : (u | 0x80000000u); +} + +static void top_k_radix_select_f32( + const float * src, + int32_t * dst_idx, + const int ncols, + const int k, + uint32_t * slm, + const sycl::nd_item<1> & item_ct1 +) { + using local_atomic = sycl::atomic_ref<uint32_t, sycl::memory_order::relaxed, + sycl::memory_scope::work_group, + sycl::access::address_space::local_space>; + + const int tid = item_ct1.get_local_id(0); + const int block_size = item_ct1.get_local_range(0); + + uint32_t * hist = slm; + uint32_t * s_digit = slm + SYCL_TOP_K_RADIX_HIST_SIZE; + uint32_t * s_bucket = slm + SYCL_TOP_K_RADIX_HIST_SIZE + 1; + uint32_t * s_need = slm + SYCL_TOP_K_RADIX_HIST_SIZE + 2; + uint32_t * s_cnt_gt = slm + SYCL_TOP_K_RADIX_HIST_SIZE + 3; + uint32_t * s_cnt_eq = slm + SYCL_TOP_K_RADIX_HIST_SIZE + 4; + + if (tid == 0) { + *s_cnt_gt = 0; + *s_cnt_eq = 0; + } + + const int copy = tid & (SYCL_TOP_K_RADIX_HIST_COPIES - 1); + + uint32_t prefix = 0; // digits fixed so far, in place + uint32_t mask = 0; // which bits of prefix are fixed + uint32_t need = (uint32_t) k; + + for (int shift = 32 - SYCL_TOP_K_RADIX_BITS; shift >= 0; shift -= SYCL_TOP_K_RADIX_BITS) { + for (int i = tid; i < SYCL_TOP_K_RADIX_HIST_SIZE; i += block_size) { + hist[i] = 0; + } + item_ct1.barrier(sycl::access::fence_space::local_space); + + for (int col = tid; col < ncols; col += block_size) { + const uint32_t key = top_k_radix_key(src[col]); + if ((key & mask) == prefix) { + const uint32_t bucket = (key >> shift) & (SYCL_TOP_K_RADIX_BUCKETS - 1); + local_atomic(hist[bucket * SYCL_TOP_K_RADIX_HIST_COPIES + copy]).fetch_add(1u); + } + } + item_ct1.barrier(sycl::access::fence_space::local_space); + + // Lane t takes bucket 255 - t, so an inclusive scan over lanes counts from the top + // bucket downward. The pivot is the unique bucket whose cumulative count first + // reaches need; the previous cumulative count is what the higher buckets contribute. + uint32_t cnt = 0; + if (tid < SYCL_TOP_K_RADIX_BUCKETS) { + const uint32_t * h = hist + (SYCL_TOP_K_RADIX_BUCKETS - 1 - tid) * SYCL_TOP_K_RADIX_HIST_COPIES; + for (int c = 0; c < SYCL_TOP_K_RADIX_HIST_COPIES; c++) { + cnt += h[c]; + } + } + const uint32_t incl = sycl::inclusive_scan_over_group(item_ct1.get_group(), cnt, sycl::plus<uint32_t>()); + + if (tid < SYCL_TOP_K_RADIX_BUCKETS && incl >= need && incl - cnt < need) { + *s_digit = (uint32_t) (SYCL_TOP_K_RADIX_BUCKETS - 1 - tid); + *s_bucket = cnt; + *s_need = need - (incl - cnt); + } + item_ct1.barrier(sycl::access::fence_space::local_space); + + const uint32_t digit = *s_digit; + const uint32_t bucket_cnt = *s_bucket; + need = *s_need; + prefix |= digit << shift; + mask |= (uint32_t) (SYCL_TOP_K_RADIX_BUCKETS - 1) << shift; + + // Every candidate in the pivot bucket is wanted: the remaining digits cannot + // change the answer, and the masked emit below is exact as it stands. + if (bucket_cnt == need) { + break; + } + // The next pass rewrites hist and s_*; the reads above must land first. + item_ct1.barrier(sycl::access::fence_space::local_space); + } + + item_ct1.barrier(sycl::access::fence_space::local_space); + + // Exactly k - need columns have (key & mask) > prefix; the first need of the pivot-equal + // columns fill the tail. Both counters live in SLM since the whole row is this group. + const uint32_t base_eq = (uint32_t) k - need; + + for (int col = tid; col < ncols; col += block_size) { + const uint32_t kp = top_k_radix_key(src[col]) & mask; + if (kp > prefix) { + const uint32_t pos = local_atomic(*s_cnt_gt).fetch_add(1u); + dst_idx[pos] = col; + } else if (kp == prefix) { + const uint32_t pos = local_atomic(*s_cnt_eq).fetch_add(1u); + if (pos < need) { + dst_idx[base_eq + pos] = col; + } + } + } +} + +static void top_k_radix_f32_sycl( + ggml_backend_sycl_context & ctx, + const float * src, + int32_t * dst_indices, + const int64_t ncols, + const int64_t nrows, + const int k, + dpct::queue_ptr main_stream +) { + GGML_ASSERT(ncols <= INT32_MAX); + + // One group per row; every pass is a strided sweep of the row, so lanes in flight is the + // only lever, and the device's own limit is the answer -- there is nothing here that + // wants a smaller group. Must still cover the 256 buckets for the scan step. + const int block_size = ggml_sycl_info().max_work_group_sizes[ctx.device]; + GGML_ASSERT(block_size >= SYCL_TOP_K_RADIX_BUCKETS); + + const sycl::range<1> block_dims(block_size); + const sycl::range<1> grid_dims(nrows); + + main_stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<uint32_t, 1> slm(sycl::range<1>(SYCL_TOP_K_RADIX_SLM_WORDS), cgh); + + cgh.parallel_for( + sycl::nd_range<1>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<1> item_ct1) { + const int row = item_ct1.get_group(0); + + top_k_radix_select_f32( + src + (int64_t) row * ncols, dst_indices + (int64_t) row * k, + (int) ncols, k, + slm.get_multi_ptr<sycl::access::decorated::no>().get(), + item_ct1); + }); + }); +} + +// One work-group owns a whole row above, which leaves the device idle whenever a graph +// has fewer rows than it has cores -- the common case at batch size 1, where the +// sparse-attention indexer and the backend sampler both top-k a single row. The kernels +// below spread one row over several groups instead. +// +// A digit pass now needs the whole row's histogram before any group can pick the pivot, +// so the per-pass state moves to global memory and the passes become separate launches: +// a work-group barrier no longer spans the row. Each group still accumulates into SLM +// and contributes 256 global atomics at the end, so global traffic is per-group, not +// per-element. The last group to finish a pass (the one whose fetch_add returns G - 1) +// does the scan for the row and clears the histogram for the next pass, which keeps the +// launch count at one per digit rather than two. +// +// Running all four digits unconditionally costs nothing in correctness: once a bucket +// holds exactly the elements still needed, later digits only extend the prefix, and the +// count of columns above that longer prefix grows by exactly as much as `need` shrinks. +// The emit below therefore stays exact whatever pass the answer settled on. + +static constexpr int SYCL_TOP_K_RADIX_ROW_DONE = SYCL_TOP_K_RADIX_BUCKETS + 0; +static constexpr int SYCL_TOP_K_RADIX_ROW_PREFIX = SYCL_TOP_K_RADIX_BUCKETS + 1; +static constexpr int SYCL_TOP_K_RADIX_ROW_MASK = SYCL_TOP_K_RADIX_BUCKETS + 2; +static constexpr int SYCL_TOP_K_RADIX_ROW_NEED = SYCL_TOP_K_RADIX_BUCKETS + 3; +static constexpr int SYCL_TOP_K_RADIX_ROW_CNT_GT = SYCL_TOP_K_RADIX_BUCKETS + 4; +static constexpr int SYCL_TOP_K_RADIX_ROW_CNT_EQ = SYCL_TOP_K_RADIX_BUCKETS + 5; +static constexpr int SYCL_TOP_K_RADIX_ROW_WORDS = SYCL_TOP_K_RADIX_BUCKETS + 6; + +// How wide the split goes is a property of the device, not of the model: enough groups to +// cover the cores, and no more. Past that the extra groups add histogram traffic without +// adding bandwidth (measured on this device: 20 and 40 groups tie, 60 and 160 lose). +// +// nsm is max_compute_units / 16, i.e. it counts an Xe core as 16 EUs. That is a core's +// width on Xe-HPG, but an Xe2 core is 8 XVEs wide, so on Battlemage the field reads half +// the cores actually present (10 for a 20-core B60). The measured curve is flat from one +// group per core to two and only falls off at three, so a factor of two covers the device +// on Xe2 and lands in the flat region on Xe-HPG. It is the one number here that a correct +// core count would remove; it was tuned on Xe2 and has not been measured on Xe-HPG. +static constexpr int SYCL_TOP_K_RADIX_GROUPS_PER_NSM = 2; +// Splitting trades one kernel for five. Below the width at which the single-group kernel +// runs longer than those four extra launches, it wins on its own; measured break-even on +// this device sits just under 64K columns. +static constexpr int SYCL_TOP_K_RADIX_MIN_SPLIT_COLS = 65536; +// A partition thinner than this cannot keep a group's sweep busy. +static constexpr int SYCL_TOP_K_RADIX_MIN_PART_COLS = 4096; + +static int top_k_radix_split_groups(const int device, const int64_t ncols, const int64_t nrows) { + const int64_t target = (int64_t) SYCL_TOP_K_RADIX_GROUPS_PER_NSM * ggml_sycl_info().devices[device].nsm; + + // One group per row already, so a graph with rows enough to cover the device gains + // nothing from splitting and would only pay the extra launches. + if (ncols < SYCL_TOP_K_RADIX_MIN_SPLIT_COLS || nrows >= target) { + return 1; + } + + const int64_t by_rows = target / nrows; // floor: never overshoot a row that is nearly covered + const int64_t by_cols = ncols / SYCL_TOP_K_RADIX_MIN_PART_COLS; + + return (int) std::max<int64_t>(1, std::min(by_rows, by_cols)); +} + +using top_k_radix_gatomic = sycl::atomic_ref<uint32_t, sycl::memory_order::relaxed, + sycl::memory_scope::device, + sycl::access::address_space::global_space>; + +static void top_k_radix_split_pass_f32( + const float * src, + uint32_t * state, + const int ncols, + const int k, + const int shift, + const bool first, + const int part, + const int nparts, + uint32_t * slm, + const sycl::nd_item<1> & item_ct1 +) { + using local_atomic = sycl::atomic_ref<uint32_t, sycl::memory_order::relaxed, + sycl::memory_scope::work_group, + sycl::access::address_space::local_space>; + + const int tid = item_ct1.get_local_id(0); + const int block_size = item_ct1.get_local_range(0); + + uint32_t * hist = slm; + uint32_t * s_last = slm + SYCL_TOP_K_RADIX_HIST_SIZE; + uint32_t * s_row = slm + SYCL_TOP_K_RADIX_HIST_SIZE + 1; // prefix, mask, need + + // The previous launch is the barrier that publishes these, so a plain load is enough. + // One lane reads them and the group takes them from SLM: a device-scope atomic load + // is uncached here, and having every work-item issue three of them off the same + // address costs more than the whole sweep below. + if (tid == 0) { + s_row[0] = first ? 0u : state[SYCL_TOP_K_RADIX_ROW_PREFIX]; + s_row[1] = first ? 0u : state[SYCL_TOP_K_RADIX_ROW_MASK]; + s_row[2] = first ? (uint32_t) k : state[SYCL_TOP_K_RADIX_ROW_NEED]; + } + + for (int i = tid; i < SYCL_TOP_K_RADIX_HIST_SIZE; i += block_size) { + hist[i] = 0; + } + item_ct1.barrier(sycl::access::fence_space::local_space); + + const uint32_t prefix = s_row[0]; + const uint32_t mask = s_row[1]; + const uint32_t need = s_row[2]; + + const int copy = tid & (SYCL_TOP_K_RADIX_HIST_COPIES - 1); + const int chunk = (ncols + nparts - 1) / nparts; + const int col0 = part * chunk; + const int col1 = std::min(ncols, col0 + chunk); + + for (int col = col0 + tid; col < col1; col += block_size) { + const uint32_t key = top_k_radix_key(src[col]); + if ((key & mask) == prefix) { + const uint32_t bucket = (key >> shift) & (SYCL_TOP_K_RADIX_BUCKETS - 1); + local_atomic(hist[bucket * SYCL_TOP_K_RADIX_HIST_COPIES + copy]).fetch_add(1u); + } + } + item_ct1.barrier(sycl::access::fence_space::local_space); + + // One global atomic per bucket per group, not per element. + for (int b = tid; b < SYCL_TOP_K_RADIX_BUCKETS; b += block_size) { + uint32_t sum = 0; + for (int c = 0; c < SYCL_TOP_K_RADIX_HIST_COPIES; c++) { + sum += hist[b * SYCL_TOP_K_RADIX_HIST_COPIES + c]; + } + if (sum) { + top_k_radix_gatomic(state[b]).fetch_add(sum); + } + } + + // Publish this group's bins, then claim the scan if this group is the row's last. + // The group-wide barrier flushes the atomics above; only the claiming lane needs the + // release, so the device-scope fence is paid once per group rather than per work-item. + item_ct1.barrier(sycl::access::fence_space::global_and_local); + if (tid == 0) { + sycl::atomic_fence(sycl::memory_order::release, sycl::memory_scope::device); + sycl::atomic_ref<uint32_t, sycl::memory_order::acq_rel, sycl::memory_scope::device, + sycl::access::address_space::global_space> done(state[SYCL_TOP_K_RADIX_ROW_DONE]); + *s_last = (done.fetch_add(1u) == (uint32_t) (nparts - 1)) ? 1u : 0u; + } + item_ct1.barrier(sycl::access::fence_space::local_space); + + if (*s_last == 0u) { + return; + } + sycl::atomic_fence(sycl::memory_order::acquire, sycl::memory_scope::device); + + // Lane t takes bucket 255 - t, so an inclusive scan counts down from the top bucket. + uint32_t cnt = 0; + if (tid < SYCL_TOP_K_RADIX_BUCKETS) { + cnt = top_k_radix_gatomic(state[SYCL_TOP_K_RADIX_BUCKETS - 1 - tid]).load(); + } + const uint32_t incl = sycl::inclusive_scan_over_group(item_ct1.get_group(), cnt, sycl::plus<uint32_t>()); + + if (tid < SYCL_TOP_K_RADIX_BUCKETS && incl >= need && incl - cnt < need) { + const uint32_t digit = (uint32_t) (SYCL_TOP_K_RADIX_BUCKETS - 1 - tid); + top_k_radix_gatomic(state[SYCL_TOP_K_RADIX_ROW_PREFIX]).store(prefix | (digit << shift)); + top_k_radix_gatomic(state[SYCL_TOP_K_RADIX_ROW_MASK]).store( + mask | ((uint32_t) (SYCL_TOP_K_RADIX_BUCKETS - 1) << shift)); + top_k_radix_gatomic(state[SYCL_TOP_K_RADIX_ROW_NEED]).store(need - (incl - cnt)); + } + item_ct1.barrier(sycl::access::fence_space::local_space); + + // Clear for the next pass; the next launch is the barrier that orders this. + for (int b = tid; b < SYCL_TOP_K_RADIX_BUCKETS; b += block_size) { + top_k_radix_gatomic(state[b]).store(0u); + } + if (tid == 0) { + top_k_radix_gatomic(state[SYCL_TOP_K_RADIX_ROW_DONE]).store(0u); + } +} + +static void top_k_radix_split_emit_f32( + const float * src, + int32_t * dst_idx, + uint32_t * state, + const int ncols, + const int k, + const int part, + const int nparts, + uint32_t * slm, + const sycl::nd_item<1> & item_ct1 +) { + using local_atomic = sycl::atomic_ref<uint32_t, sycl::memory_order::relaxed, + sycl::memory_scope::work_group, + sycl::access::address_space::local_space>; + + const int tid = item_ct1.get_local_id(0); + const int block_size = item_ct1.get_local_range(0); + + uint32_t * s_gt = slm; + uint32_t * s_eq = slm + 1; + uint32_t * s_base_gt = slm + 2; + uint32_t * s_base_eq = slm + 3; + + uint32_t * s_row = slm + 4; // prefix, mask, need + + if (tid == 0) { + *s_gt = 0; + *s_eq = 0; + s_row[0] = state[SYCL_TOP_K_RADIX_ROW_PREFIX]; + s_row[1] = state[SYCL_TOP_K_RADIX_ROW_MASK]; + s_row[2] = state[SYCL_TOP_K_RADIX_ROW_NEED]; + } + item_ct1.barrier(sycl::access::fence_space::local_space); + + const uint32_t prefix = s_row[0]; + const uint32_t mask = s_row[1]; + const uint32_t need = s_row[2]; + + // Exactly k - need columns beat the pivot; the first need pivot-equal ones fill the tail. + const uint32_t base_eq = (uint32_t) k - need; + + const int chunk = (ncols + nparts - 1) / nparts; + const int col0 = part * chunk; + const int col1 = std::min(ncols, col0 + chunk); + + // Counting first and reserving one range per group keeps the row's two counters out of + // the inner loop: a per-element global atomic on a single address serialises the whole + // emit, and at k in the thousands that alone outweighs every read the kernel does. + for (int col = col0 + tid; col < col1; col += block_size) { + const uint32_t kp = top_k_radix_key(src[col]) & mask; + if (kp > prefix) { + local_atomic(*s_gt).fetch_add(1u); + } else if (kp == prefix) { + local_atomic(*s_eq).fetch_add(1u); + } + } + item_ct1.barrier(sycl::access::fence_space::local_space); + + if (tid == 0) { + const uint32_t n_gt = *s_gt; + const uint32_t n_eq = *s_eq; + *s_base_gt = n_gt ? top_k_radix_gatomic(state[SYCL_TOP_K_RADIX_ROW_CNT_GT]).fetch_add(n_gt) : 0u; + *s_base_eq = n_eq ? top_k_radix_gatomic(state[SYCL_TOP_K_RADIX_ROW_CNT_EQ]).fetch_add(n_eq) : 0u; + *s_gt = 0; + *s_eq = 0; + } + item_ct1.barrier(sycl::access::fence_space::local_space); + + const uint32_t base_gt_g = *s_base_gt; + const uint32_t base_eq_g = *s_base_eq; + + for (int col = col0 + tid; col < col1; col += block_size) { + const uint32_t kp = top_k_radix_key(src[col]) & mask; + if (kp > prefix) { + dst_idx[base_gt_g + local_atomic(*s_gt).fetch_add(1u)] = col; + } else if (kp == prefix) { + const uint32_t pos = base_eq_g + local_atomic(*s_eq).fetch_add(1u); + if (pos < need) { + dst_idx[base_eq + pos] = col; + } + } + } +} + +static void top_k_radix_split_f32_sycl( + ggml_backend_sycl_context & ctx, + const float * src, + int32_t * dst_indices, + const int64_t ncols, + const int64_t nrows, + const int k, + const int nparts, + dpct::queue_ptr main_stream +) { + GGML_ASSERT(ncols <= INT32_MAX); + GGML_ASSERT(nparts > 1); + + const int block_size = ggml_sycl_info().max_work_group_sizes[ctx.device]; + GGML_ASSERT(block_size >= SYCL_TOP_K_RADIX_BUCKETS); + + const size_t state_words = (size_t) nrows * SYCL_TOP_K_RADIX_ROW_WORDS; + ggml_sycl_pool_alloc<uint32_t> state_alloc(ctx.pool(), state_words); + uint32_t * state = state_alloc.get(); + + // Zero histogram, done counter and both emit counters. prefix/mask/need are seeded by + // the first pass, which ignores the stored values. + // The queue is in-order, so the passes below are already ordered after this fill. + SYCL_CHECK(CHECK_TRY_ERROR(main_stream->memset(state, 0, state_words * sizeof(uint32_t)))); + + const sycl::range<1> block_dims(block_size); + const sycl::range<1> grid_dims(nrows * nparts); + + bool first = true; + for (int shift = 32 - SYCL_TOP_K_RADIX_BITS; shift >= 0; shift -= SYCL_TOP_K_RADIX_BITS) { + const bool is_first = first; + first = false; + main_stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<uint32_t, 1> slm(sycl::range<1>(SYCL_TOP_K_RADIX_HIST_SIZE + 4), cgh); + + cgh.parallel_for( + sycl::nd_range<1>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<1> item_ct1) { + const int g = item_ct1.get_group(0); + const int row = g / nparts; + const int part = g % nparts; + + top_k_radix_split_pass_f32( + src + (int64_t) row * ncols, + state + (int64_t) row * SYCL_TOP_K_RADIX_ROW_WORDS, + (int) ncols, k, shift, is_first, part, nparts, + slm.get_multi_ptr<sycl::access::decorated::no>().get(), + item_ct1); + }); + }); + } + + main_stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor<uint32_t, 1> slm(sycl::range<1>(8), cgh); + + cgh.parallel_for( + sycl::nd_range<1>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<1> item_ct1) { + const int g = item_ct1.get_group(0); + const int row = g / nparts; + const int part = g % nparts; + + top_k_radix_split_emit_f32( + src + (int64_t) row * ncols, + dst_indices + (int64_t) row * k, + state + (int64_t) row * SYCL_TOP_K_RADIX_ROW_WORDS, + (int) ncols, k, part, nparts, + slm.get_multi_ptr<sycl::access::decorated::no>().get(), + item_ct1); + }); + }); +} + +void ggml_sycl_top_k_radix( + ggml_backend_sycl_context & ctx, + const float * src, + int32_t * dst_indices, + const int64_t ncols, + const int64_t nrows, + const int k, + dpct::queue_ptr main_stream +) { + const int nparts = top_k_radix_split_groups(ctx.device, ncols, nrows); + if (nparts > 1) { + top_k_radix_split_f32_sycl(ctx, src, dst_indices, ncols, nrows, k, nparts, main_stream); + } else { + top_k_radix_f32_sycl(ctx, src, dst_indices, ncols, nrows, k, main_stream); + } +} diff --git a/ggml/src/ggml-sycl/topk-radix.hpp b/ggml/src/ggml-sycl/topk-radix.hpp new file mode 100644 index 000000000000..db479607e45d --- /dev/null +++ b/ggml/src/ggml-sycl/topk-radix.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include "common.hpp" + +// The legacy implementation uses SLM to implement sorting and top_k selection. +// SLM is limited to 128KB on Xe, which limits how much can be sorted to k<32. +// After a k=8, the radix selection becomes beneficial for most cases, because +// scan-merge has (block + 1) * k pairs of (value, index). Given normal sorting of nlog(n), +// radix-select becomes beneficial quite early. This sets it to 8 - however, the other parameters +// (columns and rows) may also be a driving factor. +// We select the legacy implementation for k below this constant because the overhead of radix select +// exceeds the benefit for very small problems +constexpr int SYCL_TOP_K_SCAN_MERGE_MAX_K = 8; + +// Top-k of every row of src, k indices per row into dst_indices, in no particular order. +// Picks between the one-group-per-row and the split-row kernel from the shape and the device. +void ggml_sycl_top_k_radix( + ggml_backend_sycl_context & ctx, + const float * src, + int32_t * dst_indices, + const int64_t ncols, + const int64_t nrows, + const int k, + dpct::queue_ptr main_stream); diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index ed5fd7de8903..909f7a78950d 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -1398,6 +1398,11 @@ vec_dot_q6_K_q8_1(const void *__restrict__ vbq, } +// NOTE: the VDR_IQ*_Q8_1_MMVQ values deliberately differ from the identically named CUDA constants +// (vecdotq.cuh): the SYCL kernels pair them with a halved qi (e.g. QI3_S/2), so the values are not +// interchangeable and must not be copied across backends. +#define VDR_IQ2_XXS_Q8_1_MMVQ 1 + static __dpct_inline__ float vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, @@ -1429,6 +1434,8 @@ vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq, #endif } +#define VDR_IQ2_XS_Q8_1_MMVQ 1 + static __dpct_inline__ float vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, @@ -1479,6 +1486,8 @@ vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq, #endif } +#define VDR_IQ2_S_Q8_1_MMVQ 1 + static __dpct_inline__ float vec_dot_iq2_s_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { @@ -1531,6 +1540,8 @@ vec_dot_iq2_s_q8_1(const void *__restrict__ vbq, #endif } +#define VDR_IQ3_XXS_Q8_1_MMVQ 1 + static __dpct_inline__ float vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, @@ -1571,6 +1582,8 @@ vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq, #endif } +#define VDR_IQ3_S_Q8_1_MMVQ 1 + static __dpct_inline__ float vec_dot_iq3_s_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, @@ -1609,6 +1622,8 @@ vec_dot_iq3_s_q8_1(const void *__restrict__ vbq, #endif } +#define VDR_IQ1_S_Q8_1_MMVQ 1 + static __dpct_inline__ float vec_dot_iq1_s_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, @@ -1637,6 +1652,8 @@ vec_dot_iq1_s_q8_1(const void *__restrict__ vbq, #endif } +#define VDR_IQ1_M_Q8_1_MMVQ 1 + static __dpct_inline__ float vec_dot_iq1_m_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { @@ -1671,6 +1688,8 @@ vec_dot_iq1_m_q8_1(const void *__restrict__ vbq, } +#define VDR_IQ4_NL_Q8_1_MMVQ 2 + static __dpct_inline__ float vec_dot_iq4_nl_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { @@ -1696,6 +1715,8 @@ vec_dot_iq4_nl_q8_1(const void *__restrict__ vbq, } +#define VDR_IQ4_XS_Q8_1_MMVQ 1 + static __dpct_inline__ float vec_dot_iq4_xs_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { diff --git a/ggml/src/ggml-vulkan/CMakeLists.txt b/ggml/src/ggml-vulkan/CMakeLists.txt index e733ad5cc984..d7dec92a63c3 100644 --- a/ggml/src/ggml-vulkan/CMakeLists.txt +++ b/ggml/src/ggml-vulkan/CMakeLists.txt @@ -62,8 +62,18 @@ if (Vulkan_FOUND) ggml_add_backend_library(ggml-vulkan ggml-vulkan.cpp ../../include/ggml-vulkan.h + ggml-vulkan-types.h + ggml-vulkan-push-constants.h + ggml-vulkan-common.h + ggml-vulkan-buffers.cpp + ggml-vulkan-debug.cpp ) + # hide symbols, so dlclosed duplicate copies cannot interpose them + set_target_properties(ggml-vulkan PROPERTIES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) + set(VULKAN_SHADER_GEN_CMAKE_ARGS "") # Test all shader extensions diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp new file mode 100644 index 000000000000..039a30b4d332 --- /dev/null +++ b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp @@ -0,0 +1,956 @@ +#include "ggml-vulkan-common.h" + +ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = { + /* .get_name = */ ggml_backend_vk_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_vk_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_vk_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_vk_buffer_type_get_alloc_size, + /* .is_host = */ NULL, +}; + +static std::vector<uint32_t> ggml_vk_find_memory_properties(const vk::PhysicalDeviceMemoryProperties* mem_props, vk::MemoryRequirements* mem_req, vk::MemoryPropertyFlags flags) { + std::vector<uint32_t> indices; + + for (uint32_t i = 0; i < mem_props->memoryTypeCount; ++i) { + vk::MemoryType memory_type = mem_props->memoryTypes[i]; + if ((mem_req->memoryTypeBits & ((uint64_t)1 << i)) && + (flags & memory_type.propertyFlags) == flags && + mem_props->memoryHeaps[memory_type.heapIndex].size >= mem_req->size) { + indices.push_back(i); + } + } + return indices; +} + +static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, const std::initializer_list<vk::MemoryPropertyFlags> & req_flags_list, + void *import_ptr = nullptr) { + VK_LOG_DEBUG("ggml_vk_create_buffer(" << device->name << ", " << size << ", " << to_string(req_flags_list.begin()[0]) << ", " << to_string(req_flags_list.begin()[req_flags_list.size()-1]) << ")"); + if (size > device->max_buffer_size) { + throw vk::OutOfDeviceMemoryError("Requested buffer size exceeds device buffer size limit"); + } + + vk_buffer buf = std::make_shared<vk_buffer_struct>(); + + if (size == 0) { + buf->size = 0; + return buf; + } + + vk::BufferUsageFlags usage_flags = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst; + vk::MemoryAllocateFlags mem_flags {}; + if (device->buffer_device_address) { + usage_flags |= vk::BufferUsageFlagBits::eShaderDeviceAddress; + mem_flags |= vk::MemoryAllocateFlagBits::eDeviceAddress; + } + + vk::BufferCreateInfo buffer_create_info{ + vk::BufferCreateFlags(), + size, + usage_flags, + vk::SharingMode::eExclusive, + 0, + nullptr, + }; + + vk::ExternalMemoryBufferCreateInfo external_memory_bci; + if (import_ptr) { + external_memory_bci.handleTypes = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT; + buffer_create_info.setPNext(&external_memory_bci); + } + + buf->buffer = device->device.createBuffer(buffer_create_info); + + vk::MemoryRequirements mem_req = device->device.getBufferMemoryRequirements(buf->buffer); + + vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties(); + + const vk::MemoryPriorityAllocateInfoEXT mem_priority_info { 1.0f }; + + vk::MemoryAllocateFlagsInfo mem_flags_info { mem_flags }; + + if (device->memory_priority) { + mem_flags_info.setPNext(&mem_priority_info); + } + + if (import_ptr) { + vk::MemoryHostPointerPropertiesEXT host_pointer_props; + try { + host_pointer_props = device->device.getMemoryHostPointerPropertiesEXT(vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT, import_ptr); + } catch (vk::SystemError& e) { + GGML_LOG_WARN("ggml_vulkan: Failed getMemoryHostPointerPropertiesEXT (%s)\n", e.what()); + device->device.destroyBuffer(buf->buffer); + return {}; + } + vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties(); + + uint32_t memory_type_idx; + vk::MemoryPropertyFlags property_flags = *req_flags_list.begin(); + for (memory_type_idx = 0; memory_type_idx < 32; ++memory_type_idx) { + if (!(host_pointer_props.memoryTypeBits & (1u << memory_type_idx))) { + continue; + } + if (!(mem_req.memoryTypeBits & (1u << memory_type_idx))) { + continue; + } + + vk::MemoryType memory_type = mem_props.memoryTypes[memory_type_idx]; + // check for visible+coherent+cached. Other flags (e.g. devicelocal) are allowed + if ((memory_type.propertyFlags & property_flags) == property_flags) { + property_flags = memory_type.propertyFlags; + break; + } + } + if (memory_type_idx == 32) { + GGML_LOG_WARN("ggml_vulkan: Memory type for host allocation not found\n"); + device->device.destroyBuffer(buf->buffer); + return {}; + } + + buf->memory_property_flags = mem_props.memoryTypes[memory_type_idx].propertyFlags; + try { + vk::ImportMemoryHostPointerInfoEXT import_info; + import_info.handleType = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT; + import_info.pHostPointer = import_ptr; + import_info.setPNext(&mem_flags_info); + buf->device_memory = device->device.allocateMemory({ size, memory_type_idx, &import_info }); + } catch (const vk::SystemError& e) { + } + } else { + for (auto it = req_flags_list.begin(); it != req_flags_list.end(); it++) { + const auto & req_flags = *it; + + const std::vector<uint32_t> memory_type_indices = ggml_vk_find_memory_properties(&mem_props, &mem_req, req_flags); + + if (memory_type_indices.empty()) { + continue; + } + + bool done = false; + + for (auto mtype_it = memory_type_indices.begin(); mtype_it != memory_type_indices.end(); mtype_it++) { + try { + buf->device_memory = device->device.allocateMemory({ mem_req.size, *mtype_it, &mem_flags_info }); + buf->memory_property_flags = mem_props.memoryTypes[*mtype_it].propertyFlags; + done = true; + break; + } catch (const vk::SystemError& e) { + // loop and retry + // during last attempt throw the exception + if (it + 1 == req_flags_list.end() && mtype_it + 1 == memory_type_indices.end()) { + device->device.destroyBuffer(buf->buffer); + throw e; + } + } + } + + if (done) { + break; + } + } + } + + if (!buf->device_memory) { + device->device.destroyBuffer(buf->buffer); + throw vk::OutOfDeviceMemoryError("No suitable memory type found"); + } + + buf->ptr = nullptr; + + if (import_ptr) { + buf->ptr = import_ptr; + } else { + if (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { + buf->ptr = device->device.mapMemory(buf->device_memory, 0, VK_WHOLE_SIZE); + } + } + + device->device.bindBufferMemory(buf->buffer, buf->device_memory, 0); + + buf->device = device; + buf->size = size; + + if (device->buffer_device_address) { + const vk::BufferDeviceAddressInfo addressInfo(buf->buffer); + buf->bda_addr = device->device.getBufferAddress(addressInfo); + } + + device->memory_logger->log_allocation(buf, size); + + return buf; +} + +vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags) { + try { + return ggml_vk_create_buffer(device, size, {req_flags, fallback_flags}); + } catch (const vk::SystemError& e) { + std::cerr << "ggml_vulkan: Memory allocation of size " << size << " failed." << std::endl; + std::cerr << "ggml_vulkan: " << e.what() << std::endl; + throw e; + } +} + +vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size) { + vk_buffer buf; + // GGML_VK_HOST_VISIBLE_VIDMEM_MAX_BYTES: keep the host-visible-device-local + // preference (the BAR) for allocations up to N bytes, and force larger ones + // straight to plain device-local. + // + // WHY. On a card without ReBAR the host-visible device-local heap IS the + // small BAR window (256 MB on an RX 480). AMD does not fail an + // oversubscribed allocation there -- it migrates to GTT -- so bulk weight + // buffers past the window silently land in system RAM and every matmul + // against them streams over PCIe (measured 2026-08-01: 190 us -> 1413 us for + // one expert group). GGML_VK_DISABLE_HOST_VISIBLE_VIDMEM avoids that, but it + // is all-or-nothing: it also pushes SMALL, frequently-rewritten buffers out + // of the BAR, and writing those then costs a staging copy + submit + fence + // instead of a memcpy (measured: a 16 KB per-request activation upload went + // 0.05 ms -> 1.69 ms, +2.44 s over a 64-token run). + // A size split gets both: small buffers keep memcpy writes, bulk weights keep + // real VRAM bandwidth. Unset (0) preserves the previous behaviour exactly. + static const size_t host_visible_max = [] { + const char * e = getenv("GGML_VK_HOST_VISIBLE_VIDMEM_MAX_BYTES"); + return (e != nullptr && e[0] != '\0') ? (size_t) strtoull(e, nullptr, 10) : (size_t) 0; + }(); + const bool force_device_local = host_visible_max != 0 && size > host_visible_max; + // GGML_VK_ALLOC_LOG=1: report every device buffer and whether it ended up + // host-visible (memcpy writes) or device-local-only (staging + submit + + // fence per write, ggml_vk_buffer_write_2d). + const bool alloc_log = getenv("GGML_VK_ALLOC_LOG") != nullptr; + try { + if (force_device_local) { + // Large: never take the BAR. Mirrors the disable_host_visible_vidmem + // branch below, including its sysmem-fallback behaviour. + if (device->allow_sysmem_fallback) { + buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); + } else { + buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + } + } else if (device->prefer_host_memory) { + buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, + vk::MemoryPropertyFlagBits::eDeviceLocal}); + } else if (device->uma) { + // On UMA, prefer host-visible memory so direct tensor borrowing works. + // If unavailable, fall back to device-local memory. + buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, + vk::MemoryPropertyFlagBits::eDeviceLocal, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); + } else if (device->disable_host_visible_vidmem) { + if (device->allow_sysmem_fallback) { + buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); + } else { + buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + } + } else { + // use rebar if available, otherwise fallback to device only visible memory + if (device->allow_sysmem_fallback) { + buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, + vk::MemoryPropertyFlagBits::eDeviceLocal, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); + } else { + buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, + vk::MemoryPropertyFlagBits::eDeviceLocal}); + } + } + if (alloc_log && buf) { + const bool hv = (bool) (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible); + std::cerr << "ggml_vulkan: alloc " << size << " bytes -> " + << (hv ? "HOST_VISIBLE (memcpy writes)" : "device-local only (staged writes)") + << (force_device_local ? " [forced by size split]" : "") << std::endl; + } + } catch (const vk::SystemError& e) { + std::cerr << "ggml_vulkan: Device memory allocation of size " << size << " failed." << std::endl; + std::cerr << "ggml_vulkan: " << e.what() << std::endl; + throw e; + } + + return buf; +} + +void ggml_vk_destroy_buffer(vk_buffer& buf) { + if (buf == nullptr) { + return; + } + + if (buf->device != nullptr) { + buf->device->memory_logger->log_deallocation(buf); + } + + buf.reset(); +} + +void * ggml_vk_host_malloc(vk_device& device, size_t size) { + VK_LOG_MEMORY("ggml_vk_host_malloc(" << size << ")"); + vk_buffer buf = ggml_vk_create_buffer(device, size, + {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); + + if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) { + fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n", + size/1024.0/1024.0); + device->device.freeMemory(buf->device_memory); + device->device.destroyBuffer(buf->buffer); + return nullptr; + } + + std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex); + device->pinned_memory.push_back(std::make_tuple(buf->ptr, size, buf)); + + return buf->ptr; +} + +void ggml_vk_host_free(vk_device& device, void* ptr) { + if (ptr == nullptr) { + return; + } + VK_LOG_MEMORY("ggml_vk_host_free(" << ptr << ")"); + std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex); + + vk_buffer buf; + size_t index; + for (size_t i = 0; i < device->pinned_memory.size(); i++) { + const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]); + const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]); + if (ptr >= addr && ptr < endr) { + buf = std::get<2>(device->pinned_memory[i]); + index = i; + break; + } + } + if (buf == nullptr) { + fprintf(stderr, "WARNING: failed to free pinned memory: memory not in map\n"); + return; + } + + ggml_vk_destroy_buffer(buf); + + device->pinned_memory.erase(device->pinned_memory.begin() + index); +} + +void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset) { + std::shared_lock<std::shared_mutex> guard(device->pinned_memory_mutex); + buf = nullptr; + buf_offset = 0; + for (size_t i = 0; i < device->pinned_memory.size(); i++) { + const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]); + const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]); + if (ptr >= addr && ptr < endr) { + buf = std::get<2>(device->pinned_memory[i]); + buf_offset = ((const uint8_t *)ptr) - addr; + break; + } + } +} + +void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size) { + if (device->sync_staging == nullptr || device->sync_staging->size < size) { + VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")"); + ggml_vk_destroy_buffer(device->sync_staging); + device->sync_staging = ggml_vk_create_buffer_check(device, size, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); + } +} + +void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size) { + if (ctx->sync_staging == nullptr || ctx->sync_staging->size < size) { + VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")"); + ggml_vk_destroy_buffer(ctx->sync_staging); + ctx->sync_staging = ggml_vk_create_buffer_check(ctx->device, size, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); + } +} + +bool ggml_vk_ensure_host_read_staging_buffer(vk_device& device, size_t size) { + if (device->host_read_staging != nullptr) { + return device->host_read_staging->size >= size && + (device->host_read_staging->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached); + } + + try { + device->host_read_staging = ggml_vk_create_buffer(device, size, + {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached}); + } catch (const vk::SystemError&) { + return false; + } + + return device->host_read_staging != nullptr && + (device->host_read_staging->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached); +} + +// Device-local (no host-visible requirement -- only Phase A writes it and +// only Phase B reads it, both on-device, in the same command buffer) growable +// scratch buffer for the fused-expert batch fast path. Returns false (leaving +// the fast path unusable for this call) only on an actual allocation failure; +// callers must fall back to the per-expert dispatch loop in that case. +bool ggml_vk_ensure_wp_fused_batch_scratch_buffer(vk_device& device, size_t size) { + if (device->wp_fused_batch_scratch != nullptr && device->wp_fused_batch_scratch->size >= size) { + return true; + } + try { + ggml_vk_destroy_buffer(device->wp_fused_batch_scratch); + device->wp_fused_batch_scratch = ggml_vk_create_buffer_check( + device, size, vk::MemoryPropertyFlagBits::eDeviceLocal); + } catch (const vk::SystemError&) { + return false; + } + return device->wp_fused_batch_scratch != nullptr; +} + +static void ggml_vk_buffer_write_nc_async(ggml_backend_vk_context * ctx, vk_context& subctx, vk_buffer& dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) { + VK_LOG_DEBUG("ggml_vk_buffer_write_nc_async(" << tensor << ")"); + GGML_ASSERT(!ggml_is_contiguous(tensor)); + // Buffer is already mapped + if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { + std::cerr << "ggml_vulkan: buffer_write_nc_async dst buffer is host_visible. Use synchronous write." << std::endl; + GGML_ABORT("fatal error"); + } + // Check if src is pinned memory + vk_buffer buf = nullptr; + size_t buf_offset = 0; + ggml_vk_host_get(ctx->device, tensor->data, buf, buf_offset); + + const uint64_t ne0 = tensor->ne[0]; + const uint64_t ne1 = tensor->ne[1]; + const uint64_t ne2 = tensor->ne[2]; + const uint64_t ne3 = tensor->ne[3]; + const uint64_t nb0 = tensor->nb[0]; + const uint64_t nb1 = tensor->nb[1]; + const uint64_t nb2 = tensor->nb[2]; + const uint64_t nb3 = tensor->nb[3]; + const ggml_type type = tensor->type; + const uint64_t ts = ggml_type_size(type); + const uint64_t bs = ggml_blck_size(type); + + const uint64_t dstnb0 = ts; + const uint64_t dstnb1 = dstnb0*(ne0/bs); + const uint64_t dstnb2 = dstnb1*ne1; + const uint64_t dstnb3 = dstnb2*ne2; + + const uint64_t ne = ggml_nelements(tensor); + + if (buf != nullptr) { + // Memory is pinned, use as staging buffer + std::vector<vk::BufferCopy> slices; + + for (uint64_t i3 = 0; i3 < ne3; i3++) { + for (uint64_t i2 = 0; i2 < ne2; i2++) { + // Find longest contiguous slice + if (ne1*nb1 == dstnb2) { + slices.push_back({ buf_offset + i3*nb3 + i2*nb2, offset + i3*dstnb3 + i2*dstnb2, dstnb2 }); + } else { + for (uint64_t i1 = 0; i1 < ne1; i1++) { + if (ne0*nb0/bs == dstnb1) { + slices.push_back({ buf_offset + i3*nb3 + i2*nb2 + i1*nb1, offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, dstnb1 }); + } else { + const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1; + const uint64_t d_off = offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1; + for (uint64_t i0 = 0; i0 < ne0; i0++) { + slices.push_back({ s_off + i0*nb0, d_off + i0*dstnb0, dstnb0 }); + } + } + } + } + } + } + + ggml_vk_sync_buffers(ctx, subctx); + subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices); + return; + } + + if (!sync_staging) { + GGML_ABORT("Asynchronous write to non-pinned memory not supported"); + } + + // Staging buffer required + vk_buffer& staging = ctx->device->sync_staging; + const uint64_t copy_size = ts*ne/bs; + ggml_vk_ensure_sync_staging_buffer(ctx->device, copy_size); + VkBufferCopy buf_copy{ 0, offset, copy_size }; + + ggml_vk_sync_buffers(ctx, subctx); + vkCmdCopyBuffer(subctx->s->buffer->buf, (VkBuffer)staging->buffer, (VkBuffer)dst->buffer, 1, &buf_copy); + + for (uint64_t i3 = 0; i3 < ne3; i3++) { + for (uint64_t i2 = 0; i2 < ne2; i2++) { + // Find longest contiguous slice + if (ne1*nb1 == dstnb2) { + deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2, dstnb2, &subctx->in_memcpys); + } else { + for (uint64_t i1 = 0; i1 < ne1; i1++) { + if (ne0*nb0/bs == dstnb1) { + deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2 + i1*nb1, dstnb1, &subctx->in_memcpys); + } else { + const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1; + const uint64_t d_off = i3*dstnb3 + i2*dstnb2 + i1*dstnb1; + for (uint64_t i0 = 0; i0 < ne0; i0++) { + deferred_memcpy((uint8_t *)staging->ptr + d_off + i0*dstnb0, (const uint8_t *) tensor->data + s_off + i0*nb0, dstnb0, &subctx->in_memcpys); + } + } + } + } + } + } +} + +bool ggml_vk_buffer_write_2d_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging) { + VK_LOG_DEBUG("ggml_vk_buffer_write_2d_async(" << width << ", " << height << ")"); + // Check if src is pinned memory + vk_buffer buf = nullptr; + size_t buf_offset = 0; + ggml_vk_host_get(dst->device, src, buf, buf_offset); + + if (buf != nullptr) { + // Memory is pinned, use as staging buffer + std::vector<vk::BufferCopy> slices(1); + if (width == spitch && width == dpitch) { + // Only do single write if stride is equal + slices[0].srcOffset = buf_offset; + slices[0].dstOffset = offset; + slices[0].size = width * height; + } else { + slices.resize(height); + for (size_t i = 0; i < height; i++) { + slices[i].srcOffset = buf_offset + i * spitch; + slices[i].dstOffset = offset + i * dpitch; + slices[i].size = width; + } + } + + ggml_vk_sync_buffers(nullptr, subctx); + subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices); + return true; + } + VK_LOG_DEBUG("STAGING"); + + if (!sync_staging) { + // copy was not handled caller needs to fall back + return false; + } + + // Staging buffer required + const size_t staging_size = width * height; + ggml_vk_ensure_sync_staging_buffer(dst->device, staging_size); + + vk_buffer& staging_buffer = dst->device->sync_staging; + + std::vector<vk::BufferCopy> slices(1); + if (width == dpitch) { + slices[0].srcOffset = 0; + slices[0].dstOffset = offset; + slices[0].size = staging_size; + } else { + slices.resize(height); + for (size_t i = 0; i < height; i++) { + slices[i].srcOffset = i * width; + slices[i].dstOffset = offset + i * dpitch; + slices[i].size = width; + } + } + + ggml_vk_sync_buffers(nullptr, subctx); + subctx->s->buffer->buf.copyBuffer(staging_buffer->buffer, dst->buffer, slices); + + if (width == spitch) { + deferred_memcpy((uint8_t *)staging_buffer->ptr, src, staging_size, &subctx->in_memcpys); + } else { + for (size_t i = 0; i < height; i++) { + deferred_memcpy((uint8_t *)staging_buffer->ptr + i * width, (const uint8_t *) src + i * spitch, width, &subctx->in_memcpys); + } + } + return true; +} + +bool ggml_vk_buffer_write_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t size, bool sync_staging) { + VK_LOG_DEBUG("ggml_vk_buffer_write_async(" << size << ")"); + return ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, size, size, size, 1, sync_staging); +} + +void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height) { + VK_LOG_DEBUG("ggml_vk_buffer_write_2d(" << width << ", " << height << ")"); + + // Invalidate the WP_VK_HOST_READ_FASTPATH read-side cache. This is a + // direct host->device write outside ggml_backend_vk_graph_compute()'s + // per-graph reset/republish dance (see the reset at the top of that + // function and the republish in its "submit || last_node" block), so if + // device->host_read_source still points at (a superset of) [offset, + // offset+width*height) in dst, a later ggml_vk_buffer_read_2d() would + // take the staged_host_read/host_cached fastpath and hand back bytes + // that predate THIS write. Concretely this bit wp-expert-worker's fused + // self-check: write_io_partial()'s ggml_backend_tensor_set() -> here -> + // left host_read_source pointing at the just-overwritten region, so the + // caller's next read_result() served the stale cached snapshot instead + // of the value this write just landed. Reset unconditionally rather than + // checking for overlap -- cheap (three field writes, no Vulkan calls) + // and always correct. + static const bool host_read_fastpath = [] { + const char * e = getenv("WP_VK_HOST_READ_FASTPATH"); + return e != nullptr && e[0] == '1'; + }(); + if (host_read_fastpath) { + dst->device->host_read_source.reset(); + dst->device->host_read_source_offset = 0; + dst->device->host_read_source_size = 0; + } + + // Buffer is already mapped + if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { + GGML_ASSERT(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent); + + if (width == spitch && width == dpitch) { + memcpy((uint8_t *)dst->ptr + offset, src, width * height); + } else { + for (size_t i = 0; i < height; i++) { + memcpy((uint8_t *)dst->ptr + offset + i * dpitch, (const uint8_t *) src + i * spitch, width); + } + } + } else { + std::lock_guard<std::recursive_mutex> guard(dst->device->mutex); + + vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool); + ggml_vk_ctx_begin(dst->device, subctx); + bool ret = ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, spitch, dpitch, width, height, true); + GGML_ASSERT(ret); + ggml_vk_ctx_end(subctx); + + for (auto& cpy : subctx->in_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } + + for (auto& mset : subctx->memsets) { + memset(mset.dst, mset.val, mset.n); + } + + ggml_vk_submit(subctx, dst->device->fence); + VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device); + dst->device->device.resetFences({ dst->device->fence }); + ggml_vk_queue_command_pools_cleanup(dst->device); + } +} + +void ggml_vk_buffer_write(vk_buffer& dst, size_t offset, const void * src, size_t size) { + VK_LOG_DEBUG("ggml_vk_buffer_write(" << size << ")"); + ggml_vk_buffer_write_2d(dst, offset, src, size, size, size, 1); +} + +bool ggml_vk_buffer_read_2d_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging) { + VK_LOG_DEBUG("ggml_vk_buffer_read_2d_async(offset=" << offset << ", width=" << width << ", height=" << height << ")"); + GGML_ASSERT(width > 0); + GGML_ASSERT(height > 0); + GGML_ASSERT(src != nullptr); + + // TODO: staging_offset is not used + + // Check if dst is pinned memory + vk_buffer buf = nullptr; + size_t buf_offset = 0; + ggml_vk_host_get(src->device, dst, buf, buf_offset); + + std::vector<vk::BufferCopy> slices(1); + if (width == spitch && width == dpitch) { + // Only do single write if stride is equal + slices[0].srcOffset = offset; + slices[0].dstOffset = buf_offset; + slices[0].size = width * height; + } else { + slices.resize(height); + for (size_t i = 0; i < height; i++) { + slices[i].srcOffset = offset + i * spitch; + slices[i].dstOffset = buf_offset + i * dpitch; + slices[i].size = width; + } + } + + if (buf != nullptr) { + // Memory is pinned, use as staging buffer + ggml_vk_sync_buffers(nullptr, subctx); + subctx->s->buffer->buf.copyBuffer(src->buffer, buf->buffer, slices); + + return true; + } + VK_LOG_DEBUG("STAGING"); + + if (!sync_staging) { + // copy was not handled caller needs to fall back + return false; + } + + // Fall back to staging buffer + const size_t staging_size = width * height; + ggml_vk_ensure_sync_staging_buffer(src->device, staging_size); + + vk_buffer& staging_buffer = src->device->sync_staging; + + std::vector<vk::BufferCopy> staging_slices(1); + if (width == spitch) { + staging_slices[0].srcOffset = offset; + staging_slices[0].dstOffset = 0; + staging_slices[0].size = staging_size; + } else { + staging_slices.resize(height); + for (size_t i = 0; i < height; i++) { + staging_slices[i].srcOffset = offset + i * spitch; + staging_slices[i].dstOffset = i * width; + staging_slices[i].size = width; + } + } + + ggml_vk_sync_buffers(nullptr, subctx); + subctx->s->buffer->buf.copyBuffer(src->buffer, staging_buffer->buffer, staging_slices); + + if (width == dpitch) { + deferred_memcpy(dst, staging_buffer->ptr, staging_size, &subctx->out_memcpys); + } else { + for (size_t i = 0; i < height; i++) { + deferred_memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) staging_buffer->ptr + i * width, width, &subctx->out_memcpys); + } + } + return true; +} + +static bool ggml_vk_buffer_read_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t size, bool sync_staging = false) { + return ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, size, size, size, 1, sync_staging); +} + +void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) { + VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")"); + + // Direct reads from host-visible device memory are opt-in because PCIe reads + // can be slower than a device copy for large results. + static const size_t direct_read_max_bytes = [] { + const char * e = getenv("GGML_VK_HOST_VISIBLE_DIRECT_READ_MAX_BYTES"); + return (e != nullptr && e[0] != '\0') ? (size_t) strtoull(e, nullptr, 10) : (size_t) 0; + }(); + const bool host_visible = + static_cast<bool>(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible); + const bool host_cached = + static_cast<bool>(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached); + const bool direct_host_read = host_visible && + (src->device->uma || + (direct_read_max_bytes != 0 && height != 0 && width <= direct_read_max_bytes / height)); + + // *** WP_VK_HOST_READ_FASTPATH -- the read half of the io-small BAR fix. *** + // + // ggml_vk_buffer_write_2d already treats a host-visible destination as a + // plain memcpy with no command buffer at all (see the eHostVisible branch + // there). The read side is not symmetric: even the direct_host_read branch + // below allocates a command buffer, records a single pipelineBarrier, + // vkQueueSubmits it and waits on a fence -- a full driver round trip for a + // 10 KB decode result. That asymmetry is the ~80 us/request ns_vk_readback + // on the RX 480. + // + // What the barrier submit actually buys is two things: (a) it makes prior + // device writes visible to the host, and (b) because the compute queue is + // in-order, its fence doubles as "wait for everything already submitted". + // (a) is redundant after ANY covering fence wait -- a fence signal operation + // carries a device->host memory domain operation, and this memory is + // asserted HOST_COHERENT below. (b) is redundant when the queue is already + // quiesced, which is exactly the wp-expert-worker's case: read_result runs + // after ggml_backend_synchronize has waited on ctx->fence. + // + // So when the queue epochs say nothing is outstanding, the whole round trip + // collapses to the same memcpy the write path uses. NO NUMERICS CHANGE -- + // the identical bytes are copied, only the wait that precedes the copy is + // dropped, and only when a stronger wait has already happened. + // + // Default OFF. The byte cap keeps large prefill readbacks on the staging + // copy (a big PCIe read from BAR is slower than a device copy into GTT). + static const bool host_read_fastpath = [] { + const char * e = getenv("WP_VK_HOST_READ_FASTPATH"); + return e != nullptr && e[0] == '1'; + }(); + static const size_t host_read_fastpath_max = [] { + const char * e = getenv("WP_VK_HOST_READ_FASTPATH_MAX_BYTES"); + return (e != nullptr && e[0] != '\0') ? (size_t) strtoull(e, nullptr, 10) : (size_t) 262144; + }(); + const bool fastpath_size_ok = height != 0 && width <= host_read_fastpath_max / height; + const bool staged_host_read = host_read_fastpath && fastpath_size_ok && !host_cached && + src == src->device->host_read_source && + offset >= src->device->host_read_source_offset && + offset - src->device->host_read_source_offset <= src->device->host_read_source_size && + width * height <= src->device->host_read_source_size - (offset - src->device->host_read_source_offset); + const vk_buffer& fastpath_src = staged_host_read ? src->device->host_read_staging : src; + const size_t fastpath_offset = staged_host_read ? offset - src->device->host_read_source_offset : offset; + if (host_read_fastpath && (host_cached || staged_host_read) && + (fastpath_src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) && + (fastpath_src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent) && + (fastpath_src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached) && + fastpath_size_ok && + ggml_vk_queue_quiesced(*src->device->compute_queue->handle)) { + if (width == spitch && width == dpitch) { + memcpy(dst, (const uint8_t *) fastpath_src->ptr + fastpath_offset, width * height); + } else { + for (size_t i = 0; i < height; i++) { + memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) fastpath_src->ptr + fastpath_offset + i * spitch, width); + } + } + return; + } + + if (direct_host_read) { + GGML_ASSERT(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent); + + std::lock_guard<std::recursive_mutex> guard(src->device->mutex); + vk_context subctx = ggml_vk_create_temporary_context(src->device->compute_queue->cmd_pool); + ggml_vk_ctx_begin(src->device, subctx); + subctx->s->buffer->buf.pipelineBarrier( + vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, + {}, + { { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite, + vk::AccessFlagBits::eHostRead } }, + {}, {}); + ggml_vk_ctx_end(subctx); + ggml_vk_submit(subctx, src->device->fence); + const uint64_t epoch = ggml_vk_queue_epoch(*src->device->compute_queue->handle); + VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), + "vk_buffer_read_2d uma waitForFences", src->device); + src->device->device.resetFences({ src->device->fence }); + // This fence covers every earlier compute-queue submission (in-order queue). + ggml_vk_queue_mark_waited(*src->device->compute_queue->handle, epoch); + ggml_vk_queue_command_pools_cleanup(src->device); + + if (width == spitch && width == dpitch) { + memcpy(dst, (const uint8_t *) src->ptr + offset, width * height); + } else { + for (size_t i = 0; i < height; i++) { + memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) src->ptr + offset + i * spitch, width); + } + } + } else { + std::lock_guard<std::recursive_mutex> guard(src->device->mutex); + + vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool); + ggml_vk_ctx_begin(src->device, subctx); + bool ret = ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, spitch, dpitch, width, height, true); + GGML_ASSERT(ret); + ggml_vk_ctx_end(subctx); + + ggml_vk_submit(subctx, src->device->fence); + VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device); + src->device->device.resetFences({ src->device->fence }); + ggml_vk_queue_command_pools_cleanup(src->device); + + for (auto& cpy : subctx->out_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } + } +} + +void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size) { + VK_LOG_DEBUG("ggml_vk_buffer_read(" << src->buffer << ", " << offset << ", " << size << ")"); + ggml_vk_buffer_read_2d(src, offset, dst, size, size, size, 1); +} + +void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) { + VK_LOG_DEBUG("ggml_vk_buffer_copy_async(" << size << ")"); + // Make sure both buffers are on same device + GGML_ASSERT(src->device == dst->device); + + VkBufferCopy bc{ src_offset, dst_offset, size }; + + vkCmdCopyBuffer(ctx->s->buffer->buf, (VkBuffer)src->buffer, (VkBuffer)dst->buffer, 1, &bc); +} + +void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) { + if (src->device == dst->device) { + std::lock_guard<std::recursive_mutex> guard(src->device->mutex); + VK_LOG_DEBUG("ggml_vk_buffer_copy(SINGLE_DEVICE, " << size << ")"); + // Copy within the device + vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool); + ggml_vk_ctx_begin(src->device, subctx); + ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size); + ggml_vk_ctx_end(subctx); + ggml_vk_submit(subctx, src->device->fence); + VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device); + src->device->device.resetFences({ src->device->fence }); + ggml_vk_queue_command_pools_cleanup(src->device); + } else { + VK_LOG_DEBUG("ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")"); + // Copy device to device + ggml_vk_ensure_sync_staging_buffer(src->device, size); + + // Copy to src staging buffer + ggml_vk_buffer_copy(src->device->sync_staging, 0, src, src_offset, size); + // Copy to dst buffer + ggml_vk_buffer_write(dst, dst_offset, src->device->sync_staging->ptr, size); + } +} + +void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size) { + VK_LOG_DEBUG("ggml_vk_buffer_memset_async(" << offset << ", " << c << ", " << size << ")"); + + if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && + dst->device->uma) { + deferred_memset((uint8_t*)dst->ptr + offset, c, size, &ctx->memsets); + return; + } + + // Fall back to GPU fillBuffer for non-UMA or non-host-visible buffers + ctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c); +} + +void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size) { + VK_LOG_DEBUG("ggml_vk_buffer_memset(" << offset << ", " << c << ", " << size << ")"); + + if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && + dst->device->uma) { + memset((uint8_t*)dst->ptr + offset, c, size); + return; + } + + std::lock_guard<std::recursive_mutex> guard(dst->device->mutex); + vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool); + ggml_vk_ctx_begin(dst->device, subctx); + subctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c); + ggml_vk_ctx_end(subctx); + + ggml_vk_submit(subctx, dst->device->fence); + VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device); + dst->device->device.resetFences({ dst->device->fence }); + ggml_vk_queue_command_pools_cleanup(dst->device); +} + +ggml_backend_buffer_i ggml_backend_vk_buffer_interface = { + /* .free_buffer = */ ggml_backend_vk_buffer_free_buffer, + /* .get_base = */ ggml_backend_vk_buffer_get_base, + /* .init_tensor = */ ggml_backend_vk_buffer_init_tensor, + /* .memset_tensor = */ ggml_backend_vk_buffer_memset_tensor, + /* .set_tensor = */ ggml_backend_vk_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_vk_buffer_get_tensor, + /* .set_tensor_2d = */ ggml_backend_vk_buffer_set_tensor_2d, + /* .get_tensor_2d = */ ggml_backend_vk_buffer_get_tensor_2d, + /* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor, + /* .clear = */ ggml_backend_vk_buffer_clear, + /* .reset = */ NULL, +}; + +vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size) { + if (!device->external_memory_host) { + return {}; + } + + uintptr_t uptr = reinterpret_cast<uintptr_t>(ptr); + if (uptr & (device->min_imported_host_pointer_alignment - 1)) { + return {}; + } + if (size & (device->min_imported_host_pointer_alignment - 1)) { + return {}; + } + + const vk::MemoryPropertyFlags property_flags = vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached; + + vk_buffer buf {}; + try { + buf = ggml_vk_create_buffer(device, size, { property_flags }, ptr); + } catch (vk::SystemError& e) { + GGML_LOG_WARN("ggml_vulkan: Failed ggml_vk_create_buffer (%s)\n", e.what()); + } + + return buf; +} + diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h new file mode 100644 index 000000000000..778e3ff2470e --- /dev/null +++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h @@ -0,0 +1,329 @@ +#pragma once +#include "ggml-vulkan-push-constants.h" + +// shared globals +extern ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface; +extern bool vk_memory_logger_enabled; +extern bool vk_perf_logger_enabled; +extern bool vk_perf_logger_concurrent; +extern bool vk_node_ts_enabled; +extern bool vk_enable_sync_logger; + +static inline bool vk_want_timestamps() { + return vk_perf_logger_enabled || vk_node_ts_enabled; +} +extern uint32_t vk_perf_logger_frequency; +extern std::string vk_pipeline_stats_filter; +extern void * const vk_ptr_base; +extern vk_instance_t vk_instance; +extern ggml_backend_buffer_i ggml_backend_vk_buffer_interface; + +// instance +vk_device ggml_vk_get_device(size_t idx); +DispatchLoaderDynamic & ggml_vk_default_dispatcher(); +void ggml_vk_instance_init(); +void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx); +int ggml_vk_get_device_count(); +void ggml_vk_get_device_description(int device, char * description, size_t description_size); +bool ggml_vk_instance_layer_settings_available(); +bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions); +bool ggml_vk_instance_debug_utils_ext_available(const std::vector<vk::ExtensionProperties> & instance_extensions); +bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev); +bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch); +uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev); +bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor); + +// shaders +void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline); +vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc); +vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc, bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, ggml_type k_type, ggml_type v_type); +uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch); +void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr); +bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type); +bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type = GGML_TYPE_F16, ggml_type v_type = GGML_TYPE_F16); + +// buffers +vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0)); +vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size); +void ggml_vk_destroy_buffer(vk_buffer& buf); +void * ggml_vk_host_malloc(vk_device& device, size_t size); +void ggml_vk_host_free(vk_device& device, void* ptr); +void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset); +void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size); +void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size); +bool ggml_vk_ensure_host_read_staging_buffer(vk_device& device, size_t size); +bool ggml_vk_ensure_wp_fused_batch_scratch_buffer(vk_device& device, size_t size); +bool ggml_vk_buffer_write_2d_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false); +bool ggml_vk_buffer_write_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t size, bool sync_staging = false); +void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height); +void ggml_vk_buffer_write(vk_buffer& dst, size_t offset, const void * src, size_t size); +bool ggml_vk_buffer_read_2d_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false); +void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height); +void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size); +void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size); +void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size); +void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size); +void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size); +vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size); + +// pipelines +uint64_t vk_tensor_offset(const ggml_tensor * tensor); +uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t); +void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx); +void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, vk_pipeline& pipeline, uint32_t n); +void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx); +void ggml_vk_submit(vk_context& ctx, vk::Fence fence); +uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyProperties>& queue_family_props, const vk::QueueFlags& required, const vk::QueueFlags& avoid, int32_t compute_index, uint32_t min_num_queues); +std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_t queue_family_index, uint32_t queue_index, vk::PipelineStageFlags&& stage_flags, bool transfer_only); +std::unique_ptr<vk_queue> ggml_vk_create_aliased_queue(vk_device& device, const std::unique_ptr<vk_queue>& source); +vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_command_pool& p); +vk_context ggml_vk_create_temporary_context(vk_command_pool& p); +void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p); +void ggml_vk_queue_command_pools_cleanup(vk_device& device); +vk_subbuffer ggml_vk_subbuffer(const ggml_backend_vk_context* ctx, const vk_buffer& buf, size_t offset = 0); +void ggml_vk_sync_buffers(ggml_backend_vk_context* ctx, vk_context& subctx); +void ggml_vk_set_event(vk_context& ctx, vk::Event& event); +void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events); +vk_subbuffer ggml_vk_tensor_subbuffer(const ggml_backend_vk_context * ctx, const ggml_tensor * tensor, bool allow_misalign = false); +void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name); +void ggml_vk_ctx_end(vk_context& ctx); +void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx, bool one_time = true); +vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx); +vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx); +bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx); +size_t ggml_vk_align_size(size_t width, size_t align); +void deferred_memcpy(void * dst, const void * src, size_t size, std::vector<vk_staging_memcpy>* memcpys = nullptr); +void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_staging_memset>* memsets = nullptr); + +// matmul +vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type); +void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t padded_n); +bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor); +vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to); +vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type); +void ggml_vk_quantize_q8_1(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_subbuffer & in, const vk_subbuffer & out, uint32_t ne); +void ggml_vk_dsv4_hc_comb(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * mixes, const ggml_tensor * scale, const ggml_tensor * base, ggml_tensor * dst); +void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * weights, ggml_tensor * dst); +void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * residual, const ggml_tensor * post, const ggml_tensor * comb, ggml_tensor * dst); +void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx); +bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx); +void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx); + +// flash-attn +void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v, const ggml_tensor * mask, const ggml_tensor * sinks, ggml_tensor * dst); + +// operators +void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, const vk_subbuffer & in, const vk_subbuffer & out); +bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor * src1, const ggml_tensor * dst); +void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst); +void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_get_rows_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx); +void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_out_prod(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_sub(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +int ggml_vk_unary_mul_op_index(ggml_unary_op op); +void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx); +void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_add_id(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst); +void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); +void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); +void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); +void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); +void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); +void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); +void ggml_vk_ssm_conv(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx); +void ggml_vk_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); +void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst); +void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_sqrt(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); +void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); +void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_cos(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_log(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_diag(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_pad(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_repeat_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_silu_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +uint32_t ggml_vk_rms_partials_size(ggml_backend_vk_context * ctx, const ggml_tensor *node); +void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, float * op_params); +void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst); +void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx); +void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx, bool backprop); +void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_topk(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_topk_qsa(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx); +void ggml_vk_sum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_sum_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_mean(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_cumsum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); +void ggml_vk_cross_entropy_loss_back(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); +void ggml_vk_argmax(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_count_equal(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_solve_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_im2col(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_im2col_3d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_timestep_embedding(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_conv_transpose_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_col2im_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx); +void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); +void ggml_vk_conv_2d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_conv_3d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_conv_2d_dw(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +void ggml_vk_leaky_relu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); + +// graph +void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx); +bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int node_idx, ggml_tensor *node_begin, int node_idx_begin, bool last_node, bool almost_ready, bool submit); +void ggml_vk_compute_forward(ggml_backend_vk_context* ctx, ggml_cgraph * cgraph, ggml_tensor* tensor, int tensor_idx, bool almost_ready); +void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx); +void ggml_vk_cleanup(ggml_backend_vk_context * ctx); +void ggml_vk_synchronize(ggml_backend_vk_context * ctx); +bool ggml_vk_is_empty(ggml_tensor * node); +bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops); +bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, int num_extra); +bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, topk_moe_mode mode); +bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx); +bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx); +bool ggml_vk_can_fuse_rms_norm_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx); +bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx); +bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool elementwise); +bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx); +uint32_t ggml_vk_fuse_multi_add(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx); +void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params); + +// backend +bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer); +void ggml_backend_vk_buffer_free_buffer(ggml_backend_buffer_t buffer); +void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t buffer); +enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor); +void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); +void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size); +void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); +void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size); +void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); +bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst); +void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value); +const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft); +ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size); +size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft); +size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft); +size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor); +void ggml_backend_vk_free(ggml_backend_t backend); +ggml_backend_reg_t ggml_backend_vk_reg(); + +// debug +int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op); + +// ggml-vulkan.cpp (residual) +bool ggml_vk_lightning_indexer_k_type_supported(ggml_type type); +void ggml_vk_print_device_fault_info(const vk_device& device); +uint64_t ggml_vk_get_node_flops(const ggml_tensor * node); +void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end); +void ggml_vk_print_device_lost_info(const vk_device& device); +size_t ggml_vk_tensor_buffer_offset(const ggml_backend_vk_context * ctx, const ggml_tensor * t); +size_t ggml_vk_descriptor_offset(size_t tensor_offset, size_t alignment, size_t type_size); +uint32_t ggml_vk_concat_unit_size(ggml_type type); +bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst); + +// fork: graph plans record into their own descriptor pools/sets +inline std::vector<vk::DescriptorPool> & ggml_vk_descriptor_pools(ggml_backend_vk_context * ctx) { + return ctx->recording_plan != nullptr ? ctx->recording_plan->descriptor_pools : ctx->descriptor_pools; +} + +inline std::vector<vk::DescriptorSet> & ggml_vk_descriptor_sets(ggml_backend_vk_context * ctx) { + return ctx->recording_plan != nullptr ? ctx->recording_plan->descriptor_sets : ctx->descriptor_sets; +} + +inline uint32_t & ggml_vk_descriptor_set_idx(ggml_backend_vk_context * ctx) { + return ctx->recording_plan != nullptr ? ctx->recording_plan->descriptor_set_idx : ctx->descriptor_set_idx; +} + +inline uint32_t & ggml_vk_pipeline_descriptor_set_requirements(ggml_backend_vk_context * ctx) { + return ctx->recording_plan != nullptr ? ctx->recording_plan->pipeline_descriptor_set_requirements : ctx->pipeline_descriptor_set_requirements; +} + +template <typename T> +inline void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list<vk_subbuffer> const& descriptor_buffers, const T &push_constants, std::array<uint32_t, 3> elements) { + const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]); + const uint32_t wg1 = CEIL_DIV(elements[1], pipeline->wg_denoms[1]); + const uint32_t wg2 = CEIL_DIV(elements[2], pipeline->wg_denoms[2]); + VK_LOG_DEBUG("ggml_vk_dispatch_pipeline(" << pipeline->name << ", {"; + for (auto& buffer : descriptor_buffers) { + std::cerr << "(" << buffer.buffer->buffer << ", " << buffer.offset << ", " << buffer.size << "), "; + } + std::cerr << "}, (" << wg0 << "," << wg1 << "," << wg2 << "))"); + GGML_ASSERT(wg0 <= ctx->device->properties.limits.maxComputeWorkGroupCount[0] && + wg1 <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && + wg2 <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]); + std::vector<vk::DescriptorSet> & descriptor_sets = ggml_vk_descriptor_sets(ctx); + uint32_t & descriptor_set_idx = ggml_vk_descriptor_set_idx(ctx); + GGML_ASSERT(descriptor_set_idx < descriptor_sets.size()); + GGML_ASSERT(descriptor_buffers.size() <= MAX_PARAMETER_COUNT); + GGML_ASSERT(pipeline->parameter_count == descriptor_buffers.size()); + GGML_ASSERT(pipeline->push_constant_size == push_constant_size(push_constants)); + + vk::DescriptorSet& descriptor_set = descriptor_sets[descriptor_set_idx++]; + std::array<vk::DescriptorBufferInfo, MAX_PARAMETER_COUNT> & descriptor_buffer_infos = + ctx->descriptor_buffer_infos; + size_t descriptor_buffer_count = 0; + for (const vk_subbuffer & buffer : descriptor_buffers) { + descriptor_buffer_infos[descriptor_buffer_count++] = buffer; + } + vk::WriteDescriptorSet write_descriptor_set{ + descriptor_set, 0, 0, pipeline->parameter_count, + vk::DescriptorType::eStorageBuffer, nullptr, descriptor_buffer_infos.data() + }; + ctx->device->device.updateDescriptorSets({ write_descriptor_set }, {}); + + if (ctx->recording_plan != nullptr) { + ggml_backend_vk_graph_plan::descriptor_update update; + update.descriptor_set = descriptor_set; + update.sources.assign(descriptor_buffers.begin(), descriptor_buffers.end()); + for (vk_subbuffer & source : update.sources) { + if (source.tensor != nullptr) { + source.buffer.reset(); + } + } + ctx->recording_plan->descriptor_updates.push_back(std::move(update)); + } + + subctx->s->buffer->buf.pushConstants(pipeline->layout, vk::ShaderStageFlagBits::eCompute, 0, push_constant_size(push_constants), push_constant_data(push_constants)); + subctx->s->buffer->buf.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline->pipeline); + subctx->s->buffer->buf.bindDescriptorSets(vk::PipelineBindPoint::eCompute, + pipeline->layout, + 0, + { descriptor_set }, + {}); + { + ggml_vk_debug_label dbg(subctx, pipeline->name, wg0, wg1, wg2); + subctx->s->buffer->buf.dispatch(wg0, wg1, wg2); + } +} + diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp new file mode 100644 index 000000000000..375562ac5eb6 --- /dev/null +++ b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp @@ -0,0 +1,1566 @@ +#include "ggml-vulkan-common.h" + +bool vk_memory_logger_enabled = false; + +bool vk_perf_logger_enabled = false; + +bool vk_perf_logger_concurrent = false; + +bool vk_node_ts_enabled = false; +bool vk_enable_sync_logger = false; + +uint32_t vk_perf_logger_frequency = 1; + +std::string vk_pipeline_stats_filter; + +void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) { + if (!vk_memory_logger_enabled) { + return; + } + std::lock_guard<std::mutex> guard(log_mutex); + vk_buffer buf = buf_ref.lock(); + const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal); + const std::string type = device ? "device" : "host"; + allocations[buf->buffer] = size; + total_device += device ? size : 0; + total_host += device ? 0 : size; + VK_LOG_MEMORY(buf->device->name << ": +" << format_size(size) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host)); +} + +void vk_memory_logger::log_deallocation(vk_buffer_ref buf_ref) { + if (buf_ref.expired() || buf_ref.lock()->size == 0 || !vk_memory_logger_enabled) { + return; + } + + std::lock_guard<std::mutex> guard(log_mutex); + vk_buffer buf = buf_ref.lock(); + const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal); + std::string type = device ? "device" : "host"; + auto it = allocations.find(buf->buffer); + if (it != allocations.end()) { + total_device -= device ? it->second : 0; + total_host -= device ? 0 : it->second; + VK_LOG_MEMORY(buf->device->name << ": -" << format_size(it->second) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host)); + allocations.erase(it); + } else { + VK_LOG_MEMORY("ERROR " << buf->device->name << ": Attempted to deallocate unknown " << type << " memory at " << buf->buffer); + } +} + +#ifdef GGML_VULKAN_CHECK_RESULTS +static size_t vk_skip_checks; +static size_t vk_output_tensor; + +static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name); +static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx); +static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx); +#endif + +#ifdef GGML_VULKAN_RUN_TESTS +static void ggml_vk_print_matrix_area(const void * data, ggml_type type, int ne0, int ne1, int i0, int i1, int i2) { + if (type != GGML_TYPE_F32 && type != GGML_TYPE_F16) { + return; + } + i0 = std::max(i0, 5); + i1 = std::max(i1, 5); + i2 = std::max(i2, 0); + fprintf(stderr, " "); + for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { + fprintf(stderr, "%7d ", idx1); + } + fprintf(stderr, "\n"); + for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) { + fprintf(stderr, "%7d: ", idx0); + for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { + if (idx0 >= 0 && idx0 < ne0 && idx1 >= 0 && idx1 < ne1) { + float val; + if (type == GGML_TYPE_F32) { + val = *((const float *) data + i2*ne1*ne0 + idx1*ne0 + idx0); + } else if (type == GGML_TYPE_F16) { + val = ggml_fp16_to_fp32(*((const ggml_fp16_t *) data + i2*ne1*ne0 + idx1*ne0 + idx0)); + } else { + GGML_ABORT("fatal error"); + } + fprintf(stderr, "% 7.2f ", val); + } else { + fprintf(stderr, " "); + } + } + fprintf(stderr, "\n"); + } +} + +template <typename X_TYPE, typename Y_TYPE> +static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, int split_k, int shader_size) { + VK_LOG_DEBUG("ggml_vk_test_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << shader_size << ")"); + const size_t x_ne = m * k * batch; + const size_t y_ne = k * n * batch; + const size_t d_ne = m * n * batch; + + ggml_type x_type = std::is_same<float, X_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16; + ggml_type y_type = std::is_same<float, Y_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16; + vk_matmul_pipeline_key mm_test_key{x_type, y_type, false, false}; + auto mm_test_it = ctx->device->pipeline_matmul.find(mm_test_key); + GGML_ASSERT(mm_test_it != ctx->device->pipeline_matmul.end() && !mm_test_it->second.empty()); + auto& mm_test_configs = mm_test_it->second; + GGML_ASSERT(shader_size >= 0 && shader_size < (int)mm_test_configs.size()); + + std::string shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_ALIGNED_" + std::to_string(shader_size); + vk_pipeline p = mm_test_configs[shader_size].aligned ? mm_test_configs[shader_size].aligned : mm_test_configs[shader_size].unaligned; + + const size_t kpad = ggml_vk_align_size(k, mm_test_configs[shader_size].align); + + if (k != kpad) { + p = mm_test_configs[shader_size].unaligned; + shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_" + std::to_string(shader_size); + } + + if (split_k > 1) { + ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it); + + if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) { + // Resize buffer + if (ctx->prealloc_split_k != nullptr) { + ggml_vk_destroy_buffer(ctx->prealloc_split_k); + } + ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + } + } + + ggml_pipeline_allocate_descriptor_sets(ctx); + + vk_buffer d_X = ggml_vk_create_buffer_check(ctx->device, sizeof(X_TYPE) * x_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + vk_buffer d_Y = ggml_vk_create_buffer_check(ctx->device, sizeof(Y_TYPE) * y_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + vk_buffer d_D = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + + X_TYPE* x = (X_TYPE *) malloc(sizeof(X_TYPE) * x_ne); + Y_TYPE* y = (Y_TYPE *) malloc(sizeof(Y_TYPE) * y_ne); + float* d = (float *) malloc(sizeof(float) * d_ne); + + for (size_t i = 0; i < x_ne; i++) { + if (std::is_same<float, X_TYPE>()) { + x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; + // x[i] = 1.0f; + // x[i] = i + 1; + // x[i] = (i % k == i / k) ? 1.0f : 0.0f; + } else if (std::is_same<ggml_fp16_t, X_TYPE>()) { + x[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f); + // x[i] = ggml_fp32_to_fp16(1.0f); + // x[i] = ggml_fp32_to_fp16(i + 1); + // x[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f); + } else { + GGML_ABORT("fatal error"); + } + } + for (size_t i = 0; i < y_ne; i++) { + if (std::is_same<float, Y_TYPE>()) { + y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; + // y[i] = (i % k == i / k) ? 1.0f : 0.0f; + // y[i] = i + 1; + } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) { + y[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f); + // y[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f); + // y[i] = ggml_fp32_to_fp16(i + 1); + } else { + GGML_ABORT("fatal error"); + } + } + + ggml_vk_buffer_write(d_X, 0, x, sizeof(X_TYPE) * k * m * batch); + ggml_vk_buffer_write(d_Y, 0, y, sizeof(Y_TYPE) * k * n * batch); + + vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool); + ggml_vk_ctx_begin(ctx->device, subctx); + for (size_t i = 0; i < num_it; i++) { + ggml_vk_matmul( + ctx, subctx, p, ggml_vk_subbuffer(ctx, d_X), ggml_vk_subbuffer(ctx, d_Y), ggml_vk_subbuffer(ctx, d_D), ggml_vk_subbuffer(ctx, ctx->prealloc_split_k), + m, n, k, + k, k, m, k*m, k*n, m*n, + split_k, batch, batch, batch, 1, 1, n + ); + } + ggml_vk_ctx_end(subctx); + + auto begin = std::chrono::high_resolution_clock::now(); + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences", ctx->device); + ctx->device->device.resetFences({ ctx->fence }); + ggml_vk_queue_command_pools_cleanup(ctx->device); + + auto end = std::chrono::high_resolution_clock::now(); + double time = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0; + + // copy dst to host + ggml_vk_buffer_read(d_D, 0, d, sizeof(float) * d_ne); + + float * d_chk = (float *) malloc(sizeof(float) * d_ne); + + ggml_init_params iparams = { + /*.mem_size =*/ 1024*1024*1024, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + + ggml_context * ggml_ctx = ggml_init(iparams); + + ggml_type src0_type; + ggml_type src1_type; + + if (std::is_same<float, X_TYPE>()) { + src0_type = GGML_TYPE_F32; + } else if (std::is_same<ggml_fp16_t, X_TYPE>()) { + src0_type = GGML_TYPE_F16; + } else { + GGML_ABORT("fatal error"); + } + if (std::is_same<float, Y_TYPE>()) { + src1_type = GGML_TYPE_F32; + } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) { + src1_type = GGML_TYPE_F16; + } else { + GGML_ABORT("fatal error"); + } + + ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, src0_type, k, m, batch); + ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, src1_type, k, n, batch); + ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml); + + src0_ggml->data = x; + src1_ggml->data = y; + tensor_ggml->data = d_chk; + + ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx); + ggml_build_forward_expand(cgraph, tensor_ggml); + + ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1); + + ggml_free(ggml_ctx); + + double avg_err = 0.0; + int first_err_n = -1; + int first_err_m = -1; + int first_err_b = -1; + + for (size_t i = 0; i < m*n*batch; i++) { + double err = std::fabs(d[i] - d_chk[i]); + avg_err += err; + + if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) { + first_err_b = i / (m * n); + first_err_n = (i % (m * n)) / m; + first_err_m = (i % (m * n)) % m; + } + } + + avg_err /= m * n; + + double tflops = 2.0*m*n*k*batch*num_it / (time / 1000.0) / (1000.0*1000.0*1000.0*1000.0); + + std::cerr << "TEST " << shname << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl; + + if (avg_err > 0.1 || std::isnan(avg_err)) { + std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl; + std::cerr << "Actual result: " << std::endl << std::endl; + ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + std::cerr << "Expected result: " << std::endl << std::endl; + ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + if (split_k > 1) { + float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k); + ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k); + + std::cerr << "d_buf0: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + std::cerr << "d_buf1: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + std::cerr << "d_buf2: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + std::cerr << "d_buf3: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + free(split_k_buf); + } + } + + free(d_chk); + + ggml_vk_command_pool_cleanup(ctx->device, ctx->compute_cmd_pool); + + ggml_vk_destroy_buffer(d_X); + ggml_vk_destroy_buffer(d_Y); + ggml_vk_destroy_buffer(d_D); + + free(x); + free(y); + free(d); +} + +static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, int i0, int i1, int i2, int i3) { + if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16) { + return; + } + i0 = std::max(i0, 5); + i1 = std::max(i1, 5); + i2 = std::max(i2, 0); + i3 = std::max(i3, 0); + fprintf(stderr, " "); + for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { + fprintf(stderr, "%7d ", idx1); + } + fprintf(stderr, "\n"); + for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) { + fprintf(stderr, "%7d: ", idx0); + for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { + if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) { + float val; + if (tensor->type == GGML_TYPE_F32) { + val = *(float *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]); + } else if (tensor->type == GGML_TYPE_F16) { + val = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0])); + } else { + GGML_ABORT("fatal error"); + } + fprintf(stderr, "% 7.2f ", val); + } else { + fprintf(stderr, " "); + } + } + fprintf(stderr, "\n"); + } +} + +static void ggml_vk_quantize_data(const float * from, void * to, size_t ne, ggml_type quant) { + ggml_quantize_chunk(quant, from, to, 0, 1, ne, nullptr); +} + +static void ggml_vk_dequantize_data(const void * from, float * to, size_t ne, ggml_type quant) { + if (quant == GGML_TYPE_F32) { + memcpy(to, from, sizeof(float) * ne); + return; + } + + const auto * tt = ggml_get_type_traits(quant); + + ggml_to_float_t dequant_fn = tt->to_float; + + dequant_fn(from, to, ne); +} + +static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) { + VK_LOG_DEBUG("ggml_vk_test_dequant(" << ne << ")"); + const size_t x_sz = sizeof(float) * ne; + const size_t x_sz_f16 = sizeof(ggml_fp16_t) * ne; + const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant); + float * x = (float *) malloc(x_sz); + void * qx = malloc(qx_sz); + vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz_f16, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + float * x_ref = (float *) malloc(x_sz); + ggml_fp16_t * x_chk = (ggml_fp16_t *) malloc(x_sz_f16); + + for (size_t i = 0; i < ne; i++) { + x[i] = rand() / (float)RAND_MAX; + } + + vk_pipeline p = ggml_vk_get_to_fp16(ctx, quant); + + ggml_vk_quantize_data(x, qx, ne, quant); + ggml_vk_dequantize_data(qx, x_ref, ne, quant); + + ggml_pipeline_request_descriptor_sets(ctx, p, 1); + + ggml_pipeline_allocate_descriptor_sets(ctx); + + ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz); + + vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool); + ggml_vk_ctx_begin(ctx->device, subctx); + const std::vector<uint32_t> pc = { 1, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne }; + ggml_vk_dispatch_pipeline(ctx, subctx, p, { vk_subbuffer{ qx_buf, 0, qx_sz }, vk_subbuffer{ x_buf, 0, x_sz_f16 } }, pc, { (uint32_t)ne, 1, 1}); + ggml_vk_ctx_end(subctx); + + auto begin = std::chrono::high_resolution_clock::now(); + + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device); + ctx->device->device.resetFences({ ctx->fence }); + ggml_vk_queue_command_pools_cleanup(ctx->device); + + auto end = std::chrono::high_resolution_clock::now(); + + double ms_dequant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0; + ggml_vk_buffer_read(x_buf, 0, x_chk, x_sz_f16); + + int first_err = -1; + + double avg_err = 0.0; + for (size_t i = 0; i < ne; i++) { + double error = std::fabs(x_ref[i] - ggml_fp16_to_fp32(x_chk[i])); + avg_err += error; + + if (first_err < 0 && error > 0.05) { + first_err = i; + } + } + + avg_err /= ne; + + std::cerr << "TEST DEQUANT " << ggml_type_name(quant) << " time=" << ms_dequant << "ms avg_err=" << avg_err << std::endl; + + if (avg_err > 0.1) { + std::cerr << "first_error = " << first_err << std::endl; + std::cerr << "Actual result: " << std::endl << std::endl; + for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) { + std::cerr << ggml_fp16_to_fp32(x_chk[i]) << ", "; + } + std::cerr << std::endl << "Expected result: " << std::endl << std::endl; + for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) { + std::cerr << x_ref[i] << ", "; + } + std::cerr << std::endl; + } + + ggml_vk_destroy_buffer(x_buf); + ggml_vk_destroy_buffer(qx_buf); + + free(x); + free(qx); + free(x_ref); + free(x_chk); +} + +// This does not work without ggml q8_1 quantization support +// +// typedef uint16_t ggml_half; +// typedef uint32_t ggml_half2; +// +// #define QK8_1 32 +// typedef struct { +// union { +// struct { +// ggml_half d; // delta +// ggml_half s; // d * sum(qs[i]) +// } GGML_COMMON_AGGR_S; +// ggml_half2 ds; +// } GGML_COMMON_AGGR_U; +// int8_t qs[QK8_1]; // quants +// } block_q8_1; +// +// static void ggml_vk_test_quantize(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) { +// VK_LOG_DEBUG("ggml_vk_test_quantize(" << ne << ")"); +// GGML_ASSERT(quant == GGML_TYPE_Q8_1); +// +// const size_t x_sz = sizeof(float) * ne; +// const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant); +// float * x = (float *) malloc(x_sz); +// block_q8_1 * qx = (block_q8_1 *)malloc(qx_sz); +// block_q8_1 * qx_res = (block_q8_1 *)malloc(qx_sz); +// vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); +// vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); +// +// for (size_t i = 0; i < ne; i++) { +// x[i] = rand() / (float)RAND_MAX; +// } +// +// vk_pipeline p = ggml_vk_get_quantize_pipeline(ctx, quant); +// +// ggml_pipeline_request_descriptor_sets(ctx, p, 1); +// +// ggml_pipeline_allocate_descriptor_sets(ctx); +// +// ggml_vk_buffer_write(x_buf, 0, x, x_sz); +// +// vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool); +// ggml_vk_ctx_begin(ctx->device, subctx); +// ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, x_buf), ggml_vk_subbuffer(ctx, qx_buf), ne); +// ggml_vk_ctx_end(subctx); +// +// auto begin = std::chrono::high_resolution_clock::now(); +// +// ggml_vk_submit(subctx, ctx->fence); +// VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_quantize waitForFences"); +// ctx->device->device.resetFences({ ctx->fence }); +// ggml_vk_queue_command_pools_cleanup(ctx->device); +// +// auto end = std::chrono::high_resolution_clock::now(); +// +// double ms_quant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0; +// ggml_vk_buffer_read(qx_buf, 0, qx, qx_sz); +// +// ggml_vk_quantize_data(x, qx_res, ne, quant); +// +// int first_err = -1; +// +// for (size_t i = 0; i < ne / 32; i++) { +// double error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d)); +// +// if (first_err < 0 && error > 0.1) { +// first_err = i; +// } +// +// error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s)); +// +// if (first_err < 0 && error > 0.1) { +// first_err = i; +// } +// +// for (size_t j = 0; j < 32; j++) { +// uint64_t error = std::abs(qx_res[i].qs[j] - qx[i].qs[j]); +// +// if (first_err < 0 && error > 1) { +// first_err = i; +// } +// } +// } +// +// std::cerr << "TEST QUANTIZE " << ggml_type_name(quant) << " time=" << ms_quant << "ms " << (first_err == -1 ? "CORRECT" : "INCORRECT") << std::endl; +// +// if (first_err != -1) { +// std::cerr << "first_error = " << first_err << std::endl; +// std::cerr << "Actual result: " << std::endl << std::endl; +// std::cout << "d=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " "; +// for (size_t j = 0; j < 32; j++) { +// std::cout << " qs" << j << "=" << (uint32_t)qx[first_err].qs[j] << " "; +// } +// std::cerr << std::endl << std::endl << "Expected result: " << std::endl << std::endl; +// std::cout << "d=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " "; +// for (size_t j = 0; j < 32; j++) { +// std::cout << " qs" << j << "=" << (uint32_t)qx_res[first_err].qs[j] << " "; +// } +// std::cerr << std::endl; +// } +// +// ggml_vk_destroy_buffer(x_buf); +// ggml_vk_destroy_buffer(qx_buf); +// +// free(x); +// free(qx); +// free(qx_res); +// } + +static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, size_t split_k, size_t shader_size, ggml_type quant, bool mmq = false) { + VK_LOG_DEBUG("ggml_vk_test_dequant_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << ggml_type_name(quant) << ")"); + const size_t x_ne = m * k * batch; + const size_t y_ne = k * n * batch; + const size_t d_ne = m * n * batch; + + ggml_type b_type = mmq ? GGML_TYPE_Q8_1 : GGML_TYPE_F32; + bool f16acc = ctx->device->fp16 && !mmq; + vk_matmul_pipeline_key dq_key{quant, b_type, false, f16acc}; + auto dq_it = ctx->device->pipeline_matmul.find(dq_key); + if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) { + if (f16acc) { + dq_key.f16acc = false; + dq_it = ctx->device->pipeline_matmul.find(dq_key); + } + } + if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) { + std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl; + return; + } + auto& dq_configs = dq_it->second; + if (shader_size >= (int)dq_configs.size()) { + std::cerr << "error: shader_size " << shader_size << " >= configs.size() " << dq_configs.size() << " for " << ggml_type_name(quant) << std::endl; + return; + } + + std::string shname = std::string(ggml_type_name(quant)) + "_ALIGNED_" + std::to_string(shader_size); + vk_pipeline p = dq_configs[shader_size].aligned ? dq_configs[shader_size].aligned : dq_configs[shader_size].unaligned; + + const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, dq_configs[shader_size].align); + + if (mmq || k != kpad) { + p = dq_configs[shader_size].unaligned; + shname = std::string(ggml_type_name(quant)) + "_" + std::to_string(shader_size); + } + + if (p == nullptr) { + std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl; + return; + } + + const size_t x_sz = sizeof(float) * x_ne; + const size_t y_sz = sizeof(float) * y_ne; + const size_t qx_sz = x_ne * ggml_type_size(quant)/ggml_blck_size(quant); + const size_t qy_sz = mmq ? y_ne * ggml_type_size(GGML_TYPE_Q8_1)/ggml_blck_size(GGML_TYPE_Q8_1) : y_sz; + const size_t d_sz = sizeof(float) * d_ne; + float * x = (float *) malloc(x_sz); + float * y = (float *) malloc(y_sz); + void * qx = malloc(qx_sz); + vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + vk_buffer y_buf = ggml_vk_create_buffer_check(ctx->device, y_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + vk_buffer qy_buf = ggml_vk_create_buffer_check(ctx->device, qy_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + vk_buffer d_buf = ggml_vk_create_buffer_check(ctx->device, d_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + float * d = (float *) malloc(d_sz); + float * d_chk = (float *) malloc(d_sz); + + for (size_t i = 0; i < x_ne; i++) { + x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; + // x[i] = (i % k == i / k) ? 1.0f : 0.0f; + // x[i] = i % k; + } + + ggml_vk_quantize_data(x, qx, x_ne, quant); + + for (size_t i = 0; i < y_ne; i++) { + y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; + // y[i] = (i % k == i / k) ? 1.0f : 0.0f; + // y[i] = i % k; + } + + if (split_k > 1) { + ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it); + + if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) { + // Resize buffer + if (ctx->prealloc_split_k != nullptr) { + ggml_vk_destroy_buffer(ctx->prealloc_split_k); + } + ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + } + } + if (mmq) { + vk_pipeline pipeline_quantize_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); + ggml_pipeline_request_descriptor_sets(ctx, pipeline_quantize_q8_1, num_it); + } + + ggml_pipeline_allocate_descriptor_sets(ctx); + + ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz); + ggml_vk_buffer_write(y_buf, 0, y, y_sz); + + vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool); + ggml_vk_ctx_begin(ctx->device, subctx); + if (mmq) { + for (size_t i = 0; i < num_it; i++) { + ggml_vk_quantize_q8_1(ctx, subctx, { y_buf, 0, y_sz }, { qy_buf, 0, qy_sz }, y_ne); + ggml_vk_matmul( + ctx, subctx, p, { qx_buf, 0, qx_sz }, { qy_buf, 0, qy_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k }, + m, n, k, + k, k, m, k*m, k*n, m*n, + split_k, batch, batch, batch, 1, 1, n + ); + } + } else { + for (size_t i = 0; i < num_it; i++) { + ggml_vk_matmul( + ctx, subctx, p, { qx_buf, 0, qx_sz }, { y_buf, 0, y_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k }, + m, n, k, + k, k, m, k*m, k*n, m*n, + split_k, batch, batch, batch, 1, 1, n + ); + } + } + ggml_vk_ctx_end(subctx); + + auto begin = std::chrono::high_resolution_clock::now(); + + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device); + ctx->device->device.resetFences({ ctx->fence }); + ggml_vk_queue_command_pools_cleanup(ctx->device); + + auto end = std::chrono::high_resolution_clock::now(); + + double time_ms = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0; + ggml_vk_buffer_read(d_buf, 0, d, d_sz); + + ggml_init_params iparams = { + /*.mem_size =*/ 1024*1024*1024, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + + ggml_context * ggml_ctx = ggml_init(iparams); + + ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, quant, k, m, batch); + ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, GGML_TYPE_F32, k, n, batch); + ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml); + + src0_ggml->data = qx; + src1_ggml->data = y; + tensor_ggml->data = d_chk; + + ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx); + ggml_build_forward_expand(cgraph, tensor_ggml); + + ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1); + + ggml_free(ggml_ctx); + + double avg_err = 0.0; + int first_err_n = -1; + int first_err_m = -1; + int first_err_b = -1; + + for (size_t i = 0; i < m*n*batch; i++) { + double err = std::fabs(d[i] - d_chk[i]); + avg_err += err; + + if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) { + first_err_b = i / (m * n); + first_err_n = (i % (m * n)) / m; + first_err_m = (i % (m * n)) % m; + } + } + + avg_err /= m * n; + + double tflops = 2.0*m*n*k*batch*num_it / (time_ms / 1000.0) / (1000.0*1000.0*1000.0*1000.0); + + std::cerr << "TEST dequant matmul " << shname; + if (mmq) { + std::cerr << " mmq"; + } + std::cerr << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time_ms / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl; + + if (avg_err > 0.01 || std::isnan(avg_err)) { + std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl; + std::cerr << "Actual result: " << std::endl << std::endl; + ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + std::cerr << std::endl; + std::cerr << "Expected result: " << std::endl << std::endl; + ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + std::cerr << "src0: " << std::endl << std::endl; + ggml_vk_print_matrix_area(x, GGML_TYPE_F32, k, m, first_err_m, first_err_n, first_err_b); + std::cerr << std::endl; + std::cerr << "src1: " << std::endl << std::endl; + ggml_vk_print_matrix_area(y, GGML_TYPE_F32, k, n, first_err_m, first_err_n, first_err_b); + + if (split_k > 1) { + float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k); + ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k); + + std::cerr << "d_buf0: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + std::cerr << "d_buf1: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + std::cerr << "d_buf2: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + std::cerr << "d_buf3: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + free(split_k_buf); + } + } + + ggml_vk_destroy_buffer(qx_buf); + ggml_vk_destroy_buffer(y_buf); + ggml_vk_destroy_buffer(qy_buf); + ggml_vk_destroy_buffer(d_buf); + + free(x); + free(qx); + free(y); + free(d); + free(d_chk); +} +#endif + +int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) { + switch (op->op) { + case GGML_OP_GET_ROWS: + return 0; + case GGML_OP_MUL_MAT: + return op->ne[1]; + case GGML_OP_MUL_MAT_ID: + case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: + return op->ne[2]; + default: + return ggml_nrows(op); + } +} + +#ifdef GGML_VULKAN_CHECK_RESULTS +static void ggml_vk_print_graph_origin(const ggml_tensor * tensor, std::vector<const ggml_tensor *>& done, int level = 0) { + if (std::find(done.begin(), done.end(), tensor) != done.end() || level > 10) { + return; + } + for (int j = 0; j < level; j++) { + std::cerr << " "; + } + std::cerr << ggml_op_name(tensor->op) << " gpu=" << (tensor->extra != nullptr) << std::endl; + + done.push_back(tensor); + + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (tensor->src[i] != nullptr) { + ggml_vk_print_graph_origin(tensor->src[i], done, level + 1); + } + } +} + +static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, const void * data, int i0, int i1, int i2, int i3) { + if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16 && tensor->type != GGML_TYPE_I32) { + return; + } + i0 = std::max(i0, 5); + i1 = std::max(i1, 5); + i2 = std::max(i2, 0); + i3 = std::max(i3, 0); + fprintf(stderr, " "); + for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { + fprintf(stderr, "%7d ", idx1); + } + fprintf(stderr, "\n"); + for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) { + fprintf(stderr, "%7d: ", idx0); + for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { + if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) { + float val; + if (tensor->type == GGML_TYPE_F32) { + val = *(const float *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]); + } else if (tensor->type == GGML_TYPE_F16) { + val = ggml_fp16_to_fp32(*(const ggml_fp16_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0])); + } else if (tensor->type == GGML_TYPE_I32) { + val = *(const int32_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]); + } else { + GGML_ABORT("fatal error"); + } + fprintf(stderr, "% 7.2f ", val); + } else { + fprintf(stderr, " "); + } + } + fprintf(stderr, "\n"); + } +} + +static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name) { + void * tensor_data = tensor->data; + + const bool is_gpu = tensor->buffer != nullptr && ggml_backend_buffer_is_vk(tensor->buffer); + + if (is_gpu) { + const size_t tensor_size = ggml_nbytes(tensor); + tensor_data = malloc(tensor_size); + + ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context; + + vk_buffer buffer_gpu = buf_ctx->dev_buffer; + ggml_vk_buffer_read(buffer_gpu, vk_tensor_offset(tensor) + tensor->view_offs, tensor_data, tensor_size); + } + + std::cerr << "TENSOR CHECK " << name << " (" << tensor->name << "): " << ggml_op_name(tensor->op) << std::endl; + std::cerr << "tensor=" << tensor << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << std::endl; + if (tensor->src[0] != nullptr) { + std::cerr << "tensor->src[0]=" << tensor->src[0] << " name=" << tensor->src[0]->name << " op=" << ggml_op_name(tensor->src[0]->op) << " type=" << ggml_type_name(tensor->src[0]->type) << " ne0=" << tensor->src[0]->ne[0] << " nb0=" << tensor->src[0]->nb[0] << " ne1=" << tensor->src[0]->ne[1] << " nb1=" << tensor->src[0]->nb[1] << " ne2=" << tensor->src[0]->ne[2] << " nb2=" << tensor->src[0]->nb[2] << " ne3=" << tensor->src[0]->ne[3] << " nb3=" << tensor->src[0]->nb[3] << std::endl; + } + if (tensor->src[1] != nullptr) { + std::cerr << "tensor->src[1]=" << tensor->src[1] << " name=" << tensor->src[1]->name << " op=" << ggml_op_name(tensor->src[1]->op) << " type=" << ggml_type_name(tensor->src[1]->type) << " ne0=" << tensor->src[1]->ne[0] << " nb0=" << tensor->src[1]->nb[0] << " ne1=" << tensor->src[1]->ne[1] << " nb1=" << tensor->src[1]->nb[1] << " ne2=" << tensor->src[1]->ne[2] << " nb2=" << tensor->src[1]->nb[2] << " ne3=" << tensor->src[1]->ne[3] << " nb3=" << tensor->src[1]->nb[3] << std::endl; + } + std::cerr << std::endl << "Result:" << std::endl; + ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0); + std::cerr << std::endl; + std::vector<const ggml_tensor *> done; + ggml_vk_print_graph_origin(tensor, done); + + if (is_gpu) { + free(tensor_data); + } +} + +void * comp_result; +size_t comp_size; +size_t comp_nb[GGML_MAX_DIMS]; +size_t check_counter = 0; +static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) { + ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops]; + if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) { + return; + } + + check_counter++; + if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) { + return; + } + + VK_LOG_DEBUG("ggml_vk_check_results_0(" << tensor->name << ")"); + + struct ggml_init_params iparams = { + /*.mem_size =*/ 2ul*1024ul*1024ul*1024ul, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, + }; + + struct ggml_context * ggml_ctx = ggml_init(iparams); + + std::array<struct ggml_tensor *, GGML_MAX_SRC> src_clone = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; + const char * srci_name[GGML_MAX_SRC] = {"src0", "src1", "src2", "src3", "src4", "src5", "src6", "src7", "src8", "src9"}; + + std::map<ggml_tensor *, ggml_tensor *> cloned_tensors; + std::vector<void *> cloned_mallocs; + + struct ggml_tensor * tensor_clone = nullptr; + + for (int f = 0; f < ctx->num_additional_fused_ops + 1; ++f) { + tensor = cgraph->nodes[tensor_idx + f]; + for (int i = 0; i < GGML_MAX_SRC; i++) { + ggml_tensor * srci = tensor->src[i]; + if (srci == nullptr) { + continue; + } + // If a src tensor has been cloned, use that one + auto it = cloned_tensors.find(srci); + if (it != cloned_tensors.end()) { + src_clone[i] = it->second; + continue; + } + ggml_tensor * srci_clone = ggml_dup_tensor(ggml_ctx, srci); + size_t srci_size = ggml_nbytes(srci); + + src_clone[i] = srci_clone; + void *src_buffer = malloc(srci_size); + cloned_mallocs.push_back(src_buffer); + + srci_clone->data = src_buffer; + if (ggml_backend_buffer_is_host(srci->buffer)) { + memcpy(srci_clone->data, srci->data, srci_size); + memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS); + } else if (ggml_backend_buffer_is_vk(srci->buffer)) { + ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)srci->buffer->context; + vk_buffer& buffer_gpu = buf_ctx->dev_buffer; + uint64_t offset = vk_tensor_offset(srci) + srci->view_offs; + if (!ggml_is_contiguous(srci) && ggml_vk_dim01_contiguous(srci)) { + for (int i3 = 0; i3 < srci->ne[3]; i3++) { + for (int i2 = 0; i2 < srci->ne[2]; i2++) { + const int idx = i3*srci->ne[2] + i2; + ggml_vk_buffer_read(buffer_gpu, offset + idx * srci->nb[2], ((char *)srci_clone->data + idx * srci_clone->nb[2]), srci->ne[1] * srci->nb[1]); + } + } + + srci_clone->nb[0] = srci->nb[0]; + srci_clone->nb[1] = srci->nb[1]; + for (int i = 2; i < GGML_MAX_DIMS; i++) { + srci_clone->nb[i] = srci_clone->nb[i - 1]*srci_clone->ne[i - 1]; + } + } else { + if (offset + srci_size >= buffer_gpu->size) { + srci_size = buffer_gpu->size - offset; + } + ggml_vk_buffer_read(buffer_gpu, offset, srci_clone->data, srci_size); + memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS); + } + } else { + GGML_ABORT("fatal error"); + } + + if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { + ggml_vk_print_tensor(srci, srci_name[i]); + } + } + + if (tensor->op == GGML_OP_FLASH_ATTN_EXT) { + const float * params = (const float *)tensor->op_params; + tensor_clone = ggml_flash_attn_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], params[0], params[1], params[2]); + if (src_clone[4]) { + ggml_flash_attn_ext_add_sinks(tensor_clone, src_clone[4]); + } + } else if (tensor->op == GGML_OP_MUL_MAT) { + tensor_clone = ggml_mul_mat(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_MUL_MAT_ID) { + tensor_clone = ggml_mul_mat_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]); + } else if (tensor->op == GGML_OP_SUB) { + tensor_clone = ggml_sub(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_MUL) { + tensor_clone = ggml_mul(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_DIV) { + tensor_clone = ggml_div(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_CONCAT) { + tensor_clone = ggml_concat(ggml_ctx, src_clone[0], src_clone[1], *(int *)tensor->op_params); + } else if (tensor->op == GGML_OP_UPSCALE) { + tensor_clone = ggml_interpolate(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], (ggml_scale_mode) tensor->op_params[0]); + } else if (tensor->op == GGML_OP_SCALE) { + const float * params = (const float *)tensor->op_params; + tensor_clone = ggml_scale_bias(ggml_ctx, src_clone[0], params[0], params[1]); + } else if (tensor->op == GGML_OP_ADD1) { + tensor_clone = ggml_add1(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_ARANGE) { + const float start = ggml_get_op_params_f32(tensor, 0); + const float stop = ggml_get_op_params_f32(tensor, 1); + const float step = ggml_get_op_params_f32(tensor, 2); + tensor_clone = ggml_arange(ggml_ctx, start, stop, step); + } else if (tensor->op == GGML_OP_FILL) { + const float value = ggml_get_op_params_f32(tensor, 0); + tensor_clone = ggml_fill(ggml_ctx, src_clone[0], value); + } else if (tensor->op == GGML_OP_SQR) { + tensor_clone = ggml_sqr(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_SQRT) { + tensor_clone = ggml_sqrt(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_SIN) { + tensor_clone = ggml_sin(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_COS) { + tensor_clone = ggml_cos(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_LOG) { + tensor_clone = ggml_log(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_TRI) { + tensor_clone = ggml_tri(ggml_ctx, src_clone[0], (ggml_tri_type)ggml_get_op_params_i32(tensor, 0)); + } else if (tensor->op == GGML_OP_DIAG) { + tensor_clone = ggml_diag(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_CLAMP) { + const float * params = (const float *)tensor->op_params; + tensor_clone = ggml_clamp(ggml_ctx, src_clone[0], params[0], params[1]); + } else if (tensor->op == GGML_OP_PAD) { + tensor_clone = ggml_pad_ext(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3], + tensor->op_params[4], tensor->op_params[5], tensor->op_params[6], tensor->op_params[7]); + } else if (tensor->op == GGML_OP_PAD_REFLECT_1D) { + tensor_clone = ggml_pad_reflect_1d(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1]); + } else if (tensor->op == GGML_OP_REPEAT) { + tensor_clone = ggml_repeat(ggml_ctx, src_clone[0], tensor); + } else if (tensor->op == GGML_OP_REPEAT_BACK) { + tensor_clone = ggml_repeat_back(ggml_ctx, src_clone[0], tensor); + } else if (tensor->op == GGML_OP_ADD) { + tensor_clone = ggml_add(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_ACC) { + tensor_clone = ggml_acc(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]); + } else if (tensor->op == GGML_OP_SET) { + tensor_clone = ggml_set(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]); + } else if (tensor->op == GGML_OP_NORM) { + tensor_clone = ggml_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params); + } else if (tensor->op == GGML_OP_GROUP_NORM) { + const float * float_params = (const float *)tensor->op_params; + tensor_clone = ggml_group_norm(ggml_ctx, src_clone[0], tensor->op_params[0], float_params[1]); + } else if (tensor->op == GGML_OP_RMS_NORM) { + tensor_clone = ggml_rms_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params); + } else if (tensor->op == GGML_OP_RMS_NORM_BACK) { + const float eps = ((float *) tensor->op_params)[0]; + tensor_clone = ggml_rms_norm_back(ggml_ctx, src_clone[0], src_clone[1], eps); + } else if (tensor->op == GGML_OP_SILU_BACK) { + tensor_clone = ggml_silu_back(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_L2_NORM) { + const float eps = ((float *) tensor->op_params)[0]; + tensor_clone = ggml_l2_norm(ggml_ctx, src_clone[0], eps); + } else if (tensor->op == GGML_OP_SINKHORN_NORM) { + const float eps = ((float *) tensor->op_params)[0]; + const int32_t iters = ggml_get_op_params_i32(tensor, 1); + tensor_clone = ggml_sinkhorn_norm(ggml_ctx, src_clone[0], eps, iters); + } else if (tensor->op == GGML_OP_SOFT_MAX) { + if (tensor->src[1] != nullptr) { + const float * params = (const float *)tensor->op_params; + tensor_clone = ggml_soft_max_ext(ggml_ctx, src_clone[0], src_clone[1], params[0], params[1]); + } else { + tensor_clone = ggml_soft_max(ggml_ctx, src_clone[0]); + } + } else if (tensor->op == GGML_OP_SOFT_MAX_BACK) { + tensor_clone = ggml_soft_max_ext_back(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); + } else if (tensor->op == GGML_OP_DIAG_MASK_INF) { + tensor_clone = ggml_diag_mask_inf(ggml_ctx, src_clone[0], tensor->op_params[0]); + } else if (tensor->op == GGML_OP_ROPE || tensor->op == GGML_OP_ROPE_BACK) { + const int n_dims = ((int32_t *) tensor->op_params)[1]; + const int mode = ((int32_t *) tensor->op_params)[2]; + //const int n_ctx_ggml = ((int32_t *) tensor->op_params)[3]; + const int n_ctx_orig_ggml = ((int32_t *) tensor->op_params)[4]; + const float freq_base = ((float *) tensor->op_params)[5]; + const float freq_scale = ((float *) tensor->op_params)[6]; + const float ext_factor = ((float *) tensor->op_params)[7]; + const float attn_factor = ((float *) tensor->op_params)[8]; + const float beta_fast = ((float *) tensor->op_params)[9]; + const float beta_slow = ((float *) tensor->op_params)[10]; + if (mode & GGML_ROPE_TYPE_MROPE) { + int32_t *sections = ((int32_t *) tensor->op_params) + 11; + if (tensor->op == GGML_OP_ROPE) { + tensor_clone = ggml_rope_multi(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } else { + tensor_clone = ggml_rope_multi_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } + } else { + if (tensor->op == GGML_OP_ROPE) { + tensor_clone = ggml_rope_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } else { + tensor_clone = ggml_rope_ext_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } + } + const int n_offs = ((int32_t *) tensor->op_params)[15]; + if (n_offs != 0) { + tensor_clone = ggml_rope_set_offset(tensor_clone, n_offs); + } + } else if (tensor->op == GGML_OP_UNARY) { + switch (ggml_get_unary_op(tensor)) { + case GGML_UNARY_OP_EXP: + tensor_clone = ggml_exp(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_EXPM1: + tensor_clone = ggml_expm1(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_ELU: + tensor_clone = ggml_elu(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_SILU: + tensor_clone = ggml_silu(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_GELU: + tensor_clone = ggml_gelu(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_GELU_ERF: + tensor_clone = ggml_gelu_erf(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_GELU_QUICK: + tensor_clone = ggml_gelu_quick(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_RELU: + tensor_clone = ggml_relu(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_XIELU: + tensor_clone = ggml_xielu(ggml_ctx, src_clone[0], 0, 0, 0, 0); + ggml_set_op_params_f32(tensor_clone, 1, ggml_get_op_params_f32(tensor, 1)); + ggml_set_op_params_f32(tensor_clone, 2, ggml_get_op_params_f32(tensor, 2)); + ggml_set_op_params_f32(tensor_clone, 3, ggml_get_op_params_f32(tensor, 3)); + ggml_set_op_params_f32(tensor_clone, 4, ggml_get_op_params_f32(tensor, 4)); + break; + case GGML_UNARY_OP_NEG: + tensor_clone = ggml_neg(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_TANH: + tensor_clone = ggml_tanh(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_SIGMOID: + tensor_clone = ggml_sigmoid(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_HARDSIGMOID: + tensor_clone = ggml_hardsigmoid(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_HARDSWISH: + tensor_clone = ggml_hardswish(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_ABS: + tensor_clone = ggml_abs(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_SOFTPLUS: + tensor_clone = ggml_softplus(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_STEP: + tensor_clone = ggml_step(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_ROUND: + tensor_clone = ggml_round(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_CEIL: + tensor_clone = ggml_ceil(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_FLOOR: + tensor_clone = ggml_floor(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_TRUNC: + tensor_clone = ggml_trunc(ggml_ctx, src_clone[0]); + break; + case GGML_UNARY_OP_SGN: + tensor_clone = ggml_sgn(ggml_ctx, src_clone[0]); + break; + default: + std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl; + GGML_ABORT("fatal error"); + } + } else if (tensor->op == GGML_OP_GLU) { + if (src_clone[1] == nullptr) { + tensor_clone = ggml_glu(ggml_ctx, src_clone[0], (ggml_glu_op) tensor->op_params[0], tensor->op_params[1]); + } else { + tensor_clone = ggml_glu_split(ggml_ctx, src_clone[0], src_clone[1], (ggml_glu_op) tensor->op_params[0]); + } + ggml_set_op_params_i32(tensor_clone, 2, ggml_get_op_params_i32(tensor, 2)); + ggml_set_op_params_i32(tensor_clone, 3, ggml_get_op_params_i32(tensor, 3)); + } else if (tensor->op == GGML_OP_CPY || tensor->op == GGML_OP_DUP) { + if (tensor->src[1] == nullptr) { + tensor_clone = ggml_dup(ggml_ctx, src_clone[0]); + tensor_clone->type = tensor->type; + } else { + tensor_clone = ggml_cpy(ggml_ctx, src_clone[0], src_clone[1]); + } + } else if (tensor->op == GGML_OP_CONT) { + tensor_clone = ggml_cont_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); + } else if (tensor->op == GGML_OP_RESHAPE) { + tensor_clone = ggml_reshape_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); + } else if (tensor->op == GGML_OP_VIEW) { + tensor_clone = ggml_view_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], tensor->nb[1], tensor->nb[2], tensor->nb[3], ((int32_t *) tensor->op_params)[0]); + } else if (tensor->op == GGML_OP_PERMUTE) { + int32_t * params = (int32_t *)tensor->op_params; + tensor_clone = ggml_permute(ggml_ctx, src_clone[0], params[0], params[1], params[2], params[3]); + } else if (tensor->op == GGML_OP_TRANSPOSE) { + tensor_clone = ggml_transpose(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_GET_ROWS) { + tensor_clone = ggml_get_rows(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_ARGSORT) { + tensor_clone = ggml_argsort(ggml_ctx, src_clone[0], (ggml_sort_order) *(int *)tensor->op_params); + } else if (tensor->op == GGML_OP_TOP_K) { + tensor_clone = ggml_top_k(ggml_ctx, src_clone[0], tensor->ne[0]); + } else if (tensor->op == GGML_OP_SUM) { + tensor_clone = ggml_sum(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_SUM_ROWS) { + tensor_clone = ggml_sum_rows(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_CUMSUM) { + tensor_clone = ggml_cumsum(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_DSV4_HC_COMB) { + tensor_clone = ggml_dsv4_hc_comb(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], + ggml_get_op_params_f32(tensor, 0), ggml_get_op_params_i32(tensor, 1)); + } else if (tensor->op == GGML_OP_DSV4_HC_PRE) { + if (ggml_get_op_params_i32(tensor, 1) != 0) { + tensor_clone = ggml_dsv4_hc_pre_gated(ggml_ctx, src_clone[0], src_clone[1], ggml_get_op_params_f32(tensor, 0)); + } else { + tensor_clone = ggml_dsv4_hc_pre(ggml_ctx, src_clone[0], src_clone[1]); + } + } else if (tensor->op == GGML_OP_DSV4_HC_POST) { + tensor_clone = ggml_dsv4_hc_post(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]); + } else if (tensor->op == GGML_OP_MEAN) { + tensor_clone = ggml_mean(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_ARGMAX) { + tensor_clone = ggml_argmax(ggml_ctx, src_clone[0]); + } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS) { + tensor_clone = ggml_cross_entropy_loss(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS_BACK) { + tensor_clone = ggml_cross_entropy_loss_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]); + } else if (tensor->op == GGML_OP_COUNT_EQUAL) { + tensor_clone = ggml_count_equal(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_SOLVE_TRI) { + tensor_clone = ggml_solve_tri(ggml_ctx, src_clone[0], src_clone[1], true, true, false); + } else if (tensor->op == GGML_OP_IM2COL) { + const int32_t s0 = tensor->op_params[0]; + const int32_t s1 = tensor->op_params[1]; + const int32_t p0 = tensor->op_params[2]; + const int32_t p1 = tensor->op_params[3]; + const int32_t d0 = tensor->op_params[4]; + const int32_t d1 = tensor->op_params[5]; + + const bool is_2D = tensor->op_params[6] == 1; + tensor_clone = ggml_im2col(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1, is_2D, tensor->type); + } else if (tensor->op == GGML_OP_IM2COL_3D) { + const int32_t s0 = tensor->op_params[0]; + const int32_t s1 = tensor->op_params[1]; + const int32_t s2 = tensor->op_params[2]; + const int32_t p0 = tensor->op_params[3]; + const int32_t p1 = tensor->op_params[4]; + const int32_t p2 = tensor->op_params[5]; + const int32_t d0 = tensor->op_params[6]; + const int32_t d1 = tensor->op_params[7]; + const int32_t d2 = tensor->op_params[8]; + const int32_t IC = tensor->op_params[9]; + + tensor_clone = ggml_im2col_3d(ggml_ctx, src_clone[0], src_clone[1], IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, tensor->type); + } else if (tensor->op == GGML_OP_TIMESTEP_EMBEDDING) { + const int32_t dim = tensor->op_params[0]; + const int32_t max_period = tensor->op_params[1]; + tensor_clone = ggml_timestep_embedding(ggml_ctx, src_clone[0], dim, max_period); + } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_1D){ + const int32_t s0 = tensor->op_params[0]; + const int32_t p0 = tensor->op_params[1]; + const int32_t d0 = tensor->op_params[2]; + tensor_clone = ggml_conv_transpose_1d(ggml_ctx, src_clone[0], src_clone[1], s0, p0, d0); + } else if (tensor->op == GGML_OP_COL2IM_1D) { + const int32_t stride = tensor->op_params[0]; + const int32_t oc = tensor->op_params[1]; + const int32_t p0 = tensor->op_params[2]; + tensor_clone = ggml_col2im_1d(ggml_ctx, src_clone[0], stride, oc, p0); + } else if (tensor->op == GGML_OP_POOL_1D) { + enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]); + const int32_t k0 = tensor->op_params[1]; + const int32_t s0 = tensor->op_params[2]; + const int32_t p0 = tensor->op_params[3]; + + tensor_clone = ggml_pool_1d(ggml_ctx, src_clone[0], op, k0, s0, p0); + } else if (tensor->op == GGML_OP_POOL_2D) { + enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]); + const int32_t k0 = tensor->op_params[1]; + const int32_t k1 = tensor->op_params[2]; + const int32_t s0 = tensor->op_params[3]; + const int32_t s1 = tensor->op_params[4]; + const int32_t p0 = tensor->op_params[5]; + const int32_t p1 = tensor->op_params[6]; + + tensor_clone = ggml_pool_2d(ggml_ctx, src_clone[0], op, k0, k1, s0, s1, p0, p1); + } else if (tensor->op == GGML_OP_CONV_2D) { + const int32_t s0 = tensor->op_params[0]; + const int32_t s1 = tensor->op_params[1]; + const int32_t p0 = tensor->op_params[2]; + const int32_t p1 = tensor->op_params[3]; + const int32_t d0 = tensor->op_params[4]; + const int32_t d1 = tensor->op_params[5]; + tensor_clone = ggml_conv_2d(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1); + } else if (tensor->op == GGML_OP_CONV_3D) { + const int32_t s0 = tensor->op_params[0]; + const int32_t s1 = tensor->op_params[1]; + const int32_t s2 = tensor->op_params[2]; + const int32_t p0 = tensor->op_params[3]; + const int32_t p1 = tensor->op_params[4]; + const int32_t p2 = tensor->op_params[5]; + const int32_t d0 = tensor->op_params[6]; + const int32_t d1 = tensor->op_params[7]; + const int32_t d2 = tensor->op_params[8]; + const int32_t IC = tensor->op_params[9]; + const int32_t N = tensor->op_params[10]; + const int32_t OC = tensor->op_params[11]; + tensor_clone = ggml_conv_3d_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, s2, p0, p1, p2, d0, d1, d2, IC, N, OC); + } else if (tensor->op == GGML_OP_CONV_2D_DW) { + const int32_t s0 = tensor->op_params[0]; + const int32_t s1 = tensor->op_params[1]; + const int32_t p0 = tensor->op_params[2]; + const int32_t p1 = tensor->op_params[3]; + const int32_t d0 = tensor->op_params[4]; + const int32_t d1 = tensor->op_params[5]; + tensor_clone = ggml_conv_2d_dw_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1); + } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_2D) { + const int32_t s = tensor->op_params[0]; + tensor_clone = ggml_conv_transpose_2d_p0(ggml_ctx, src_clone[0], src_clone[1], s); + } else if (tensor->op == GGML_OP_LEAKY_RELU) { + const float * op_params = (const float *)tensor->op_params; + tensor_clone = ggml_leaky_relu(ggml_ctx, src_clone[0], op_params[0], false); + } else if (tensor->op == GGML_OP_RWKV_WKV6) { + tensor_clone = ggml_rwkv_wkv6(ggml_ctx, src_clone[0], src_clone[1], + src_clone[2], src_clone[3], src_clone[4], src_clone[5]); + } else if (tensor->op == GGML_OP_RWKV_WKV7) { + tensor_clone = ggml_rwkv_wkv7(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], + src_clone[4], src_clone[5], src_clone[6]); + } else if (tensor->op == GGML_OP_GATED_LINEAR_ATTN) { + const float * op_params = (const float *)tensor->op_params; + tensor_clone = ggml_gated_linear_attn(ggml_ctx, src_clone[0], src_clone[1], + src_clone[2], src_clone[3], src_clone[4], op_params[0]); + } else if (tensor->op == GGML_OP_LIGHTNING_INDEXER) { + tensor_clone = ggml_lightning_indexer(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]); + } else if (tensor->op == GGML_OP_GATED_DELTA_NET) { + tensor_clone = ggml_gated_delta_net(ggml_ctx, src_clone[0], src_clone[1], + src_clone[2], src_clone[3], src_clone[4], src_clone[5], + ggml_get_op_params_i32(tensor, 0)); + } else if (tensor->op == GGML_OP_OPT_STEP_ADAMW) { + src_clone[0]->flags = tensor->src[0]->flags; + tensor_clone = ggml_opt_step_adamw(ggml_ctx, src_clone[0], src_clone[1], + src_clone[2], src_clone[3], src_clone[4]); + } else if (tensor->op == GGML_OP_OPT_STEP_SGD) { + src_clone[0]->flags = tensor->src[0]->flags; + tensor_clone = ggml_opt_step_sgd(ggml_ctx, src_clone[0], src_clone[1], + src_clone[2]); + } else if (tensor->op == GGML_OP_ADD_ID) { + tensor_clone = ggml_add_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]); + } else if (tensor->op == GGML_OP_SSM_SCAN) { + const int32_t K = ggml_get_op_params_i32(tensor, 0); + tensor_clone = ggml_ssm_scan(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], + src_clone[3], src_clone[4], src_clone[5], src_clone[6], K); + } else if (tensor->op == GGML_OP_SSM_CONV) { + tensor_clone = ggml_ssm_conv(ggml_ctx, src_clone[0], src_clone[1]); + } else if (tensor->op == GGML_OP_ROLL) { + const int32_t s0 = tensor->op_params[0]; + const int32_t s1 = tensor->op_params[1]; + const int32_t s2 = tensor->op_params[2]; + const int32_t s3 = tensor->op_params[3]; + tensor_clone = ggml_roll(ggml_ctx, src_clone[0], s0, s1, s2, s3); + } + else { + std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl; + GGML_ABORT("fatal error"); + } + cloned_tensors[tensor] = tensor_clone; + } + + ggml_cgraph * cgraph_cpu = ggml_new_graph(ggml_ctx); + ggml_build_forward_expand(cgraph_cpu, tensor_clone); + + ggml_graph_compute_with_ctx(ggml_ctx, cgraph_cpu, 8); + + if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { + ggml_vk_print_tensor(tensor_clone, "tensor_clone"); + } + + comp_size = ggml_nbytes(tensor_clone); + + comp_result = malloc(comp_size); + memcpy(comp_result, tensor_clone->data, comp_size); + memcpy(comp_nb, tensor_clone->nb, sizeof(size_t) * GGML_MAX_DIMS); + + for (auto m : cloned_mallocs) { + free(m); + } + + ggml_free(ggml_ctx); + + VK_LOG_DEBUG("END ggml_vk_check_results_0(" << tensor->name << ")"); +} + +static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) { + ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops]; + if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) { + return; + } + + if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) { + return; + } + + VK_LOG_DEBUG("ggml_vk_check_results_1(" << tensor->name << ")"); + + ggml_tensor * src0 = tensor->src[0]; + ggml_tensor * src1 = tensor->src[1]; + ggml_tensor * src2 = tensor->src[2]; + ggml_tensor * src3 = tensor->src[3]; + + void * tensor_data = tensor->data; + + if (ggml_backend_buffer_is_vk(tensor->buffer)) { + size_t tensor_size = ggml_nbytes(tensor); + tensor_data = malloc(tensor_size); + + ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context; + + vk_buffer& buffer_gpu = buf_ctx->dev_buffer; + uint64_t offset = vk_tensor_offset(tensor) + tensor->view_offs; + if (offset + tensor_size >= buffer_gpu->size) { + tensor_size = buffer_gpu->size - offset; + } + + ggml_vk_buffer_read(buffer_gpu, offset, tensor_data, tensor_size); + } + + float first_error_result = -1.0f; + float first_error_correct = -1.0f; + std::array<int, 4> first_error = { -1, -1, -1, -1 }; + double avg_err = 0.0; + size_t counter = 0; + + for (int i3 = 0; i3 < tensor->ne[3]; i3++) { + for (int i2 = 0; i2 < tensor->ne[2]; i2++) { + for (int i1 = 0; i1 < tensor->ne[1]; i1++) { + for (int i0 = 0; i0 < tensor->ne[0]; i0++) { + const bool buffer_size_fit = i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0] < comp_size; + float correct = 0.0f; + float result = 0.0f; + + if (buffer_size_fit) { + if (tensor->type == GGML_TYPE_F32) { + correct = *(float *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]); + result = *(float *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]); + } else if (tensor->type == GGML_TYPE_F16) { + correct = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0])); + result = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0])); + } else if (tensor->type == GGML_TYPE_BF16) { + correct = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0])); + result = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0])); + } else if (tensor->type == GGML_TYPE_I32) { + correct = *(int32_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]); + result = *(int32_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]); + } else if (tensor->type == GGML_TYPE_I64) { + correct = *(int64_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]); + result = *(int64_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]); + } else { + std::cerr << "Results check not implemented for type " << ggml_type_name(tensor->type) << std::endl; + } + } else { + std::cerr << "Missing debug code for type " << ggml_type_name(tensor->type) << std::endl; + GGML_ABORT("fatal error"); + } + + if ((std::isnan(correct) != std::isnan(result)) || (std::isinf(correct) != std::isinf(result)) || !buffer_size_fit) { + std::cerr << "ERROR: Invalid value in " << ggml_op_name(tensor->op) << " i3=" << i3 << " i2=" << i2 << " i1=" << i1 << " i0=" << i0 << " result=" << result << " correct=" << correct << " avg_err=" << (avg_err / counter) << std::endl; + std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; + if (src0 != nullptr) { + std::cerr << "src0=" << src0 << " src0->name=" << src0->name << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; + } + if (src1 != nullptr) { + std::cerr << "src1=" << src1 << " src1->name=" << src1->name << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; + } + if (src2 != nullptr) { + std::cerr << "src2=" << src2 << " src2->name=" << src2->name << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl; + } + if (src3 != nullptr) { + std::cerr << "src3=" << src3 << " src3->name=" << src3->name << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl; + } + std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl; + std::cerr << std::endl << "Result:" << std::endl; + ggml_vk_print_tensor_area(tensor, tensor_data, i0, i1, i2, i3); + std::cerr << std::endl << "Correct:" << std::endl; + ggml_vk_print_tensor_area(tensor, comp_result, i0, i1, i2, i3); + std::cerr << std::endl; + std::vector<const ggml_tensor *> done; + ggml_vk_print_graph_origin(tensor, done); + GGML_ABORT("fatal error"); + } + const double denom = std::fabs(correct) > 1.0f ? (std::fabs(correct) > 1e-8 ? std::fabs(correct) : 1e-8) : 1.0f; + if (first_error[0] == -1 && std::fabs(correct - result) / denom > 0.5) { + first_error[0] = i0; + first_error[1] = i1; + first_error[2] = i2; + first_error[3] = i3; + first_error_result = result; + first_error_correct = correct; + } + + // Special case, value is infinite, avoid NaN result in avg_err + // NaN also appears in results, if both are nan error is 0 + if (!std::isinf(correct) && !std::isinf(result) && !std::isnan(correct) && !std::isnan(result)) { + avg_err += std::fabs(correct - result) / denom; + } + counter++; + } + } + } + } + + avg_err /= counter; + + if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { + std::cerr << "TENSOR CHECK: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl; + std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; + if (src0 != nullptr) { + std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; + } + if (src1 != nullptr) { + std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; + } + if (src2 != nullptr) { + std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl; + } + if (src3 != nullptr) { + std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl; + } + std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl; + std::cerr << std::endl << "Result:" << std::endl; + ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0); + std::cerr << std::endl << "Correct:" << std::endl; + ggml_vk_print_tensor_area(tensor, comp_result, 5, 5, 0, 0); + std::cerr << std::endl; + std::vector<const ggml_tensor *> done; + ggml_vk_print_graph_origin(tensor, done); + } + + if (avg_err > 0.01 || std::isnan(avg_err)) { + std::cerr << "ERROR: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl; + std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; + if (src0 != nullptr) { + std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; + } + if (src1 != nullptr) { + std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; + } + if (src2 != nullptr) { + std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl; + } + if (src3 != nullptr) { + std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl; + } + std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl; + std::cerr << std::endl << "Result:" << std::endl; + ggml_vk_print_tensor_area(tensor, tensor_data, first_error[0], first_error[1], first_error[2], first_error[3]); + std::cerr << std::endl << "Correct:" << std::endl; + ggml_vk_print_tensor_area(tensor, comp_result, first_error[0], first_error[1], first_error[2], first_error[3]); + std::cerr << std::endl; + std::vector<const ggml_tensor *> done; + ggml_vk_print_graph_origin(tensor, done); + GGML_ABORT("fatal error"); + } else { + std::cerr << check_counter << " " << tensor->name << " op=" << ggml_op_name(tensor->op) << " avg_err=" << avg_err << std::endl; + } + + free(comp_result); + comp_result = nullptr; + comp_size = 0; + + if (ggml_backend_buffer_is_vk(tensor->buffer)) { + free(tensor_data); + } + + VK_LOG_DEBUG("END ggml_vk_check_results_1(" << tensor->name << ")"); +} +#endif + diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h new file mode 100644 index 000000000000..cf8d7eb6f0ab --- /dev/null +++ b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h @@ -0,0 +1,1224 @@ +#pragma once +#include "ggml-vulkan-types.h" + +uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t); + +uint32_t ggml_vk_concat_unit_size(ggml_type type); + +struct vk_mat_mat_push_constants { + uint32_t M; uint32_t N; uint32_t K; + uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; + uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d; + uint32_t base_work_group_z; uint32_t num_batches; + uint32_t k_split; + uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3; + uint32_t padded_N; +}; + +struct vk_mat_vec_push_constants { + uint32_t ncols; + uint32_t stride_a; + uint32_t stride_b; + uint32_t stride_d; + uint32_t batch_stride_a; + uint32_t batch_stride_b; + uint32_t batch_stride_d; + uint32_t fusion_flags; + uint32_t base_work_group_y; + uint32_t ne02; + uint32_t ne12; + uint32_t broadcast2; + uint32_t broadcast3; +}; + +struct vk_mat_vec_p021_push_constants { + uint32_t ncols_x; + uint32_t nrows_x; + uint32_t nchannels_x; + uint32_t nchannels_y; + uint32_t b_offset; + uint32_t d_offset; + uint32_t fusion_flags; +}; + +struct vk_mat_vec_nc_push_constants { + uint32_t ncols_x; + uint32_t nrows_x; + uint32_t row_stride_x; + uint32_t channel_stride_x; + uint32_t channel_stride_y; + uint32_t channel_x_divisor; + uint32_t ne12; + uint32_t b_offset; + uint32_t d_offset; + uint32_t nb03; + uint32_t nb13; + uint32_t nb23; + uint32_t fusion_flags; +}; + +struct vk_mat_mat_id_push_constants { + uint32_t M; uint32_t N; uint32_t K; + uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; + uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d; + uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11; + uint32_t n_experts; + uint32_t hoist_row_ids; + // Weight paging. paged=0 keeps stock uniform-stride addressing so the + // non-paged path is bit-identical. paged=1 means binding 0 is the pager's + // pool and binding 5 holds each expert's offset within it, in ELEMENTS. + uint32_t paged; +}; + +struct vk_mat_vec_id_push_constants { + uint32_t ncols; + uint32_t stride_a; + uint32_t stride_b; + uint32_t stride_d; + uint32_t batch_stride_a; + uint32_t batch_stride_b; + uint32_t batch_stride_d; + uint32_t fusion_flags; + uint32_t nei0; + uint32_t ne11; + uint32_t expert_i1; + uint32_t nbi1; + // Weight paging. paged=0 keeps the stock uniform-stride addressing, so the + // non-paged path is bit-identical. paged=1 means binding 0 is the pager's + // pool and binding 6 holds each expert's block offset within it. + uint32_t paged; +}; + +struct vk_flash_attn_push_constants { + uint32_t N; + uint32_t KV; + + uint32_t ne1; + uint32_t ne2; + uint32_t ne3; + + uint32_t neq2; + uint32_t neq3; + uint32_t nek2; + uint32_t nek3; + uint32_t nev2; + uint32_t nev3; + uint32_t nem1; + uint32_t nem2; + uint32_t nem3; + + uint32_t nb01; + uint32_t nb02; + uint32_t nb03; + uint32_t nb11; + uint32_t nb12; + uint32_t nb13; + uint32_t nb21; + uint32_t nb22; + uint32_t nb23; + + float scale; + float max_bias; + float logit_softcap; + + uint32_t mask_n_head_log2; + float m0; + float m1; + + uint32_t gqa_ratio; + uint32_t split_kv; + uint32_t k_num; +}; + +static_assert(sizeof(vk_flash_attn_push_constants) <= 128, "sizeof(vk_flash_attn_push_constants) must be <= 128"); + +struct vk_fa_xe_opt_push_constants { + uint32_t kv_seq_len; + uint32_t activation_length; + uint32_t q_head; + uint32_t kv_head; + uint32_t qk_ratio; + uint32_t qk_sub_groups; + uint32_t flag; + uint32_t nbkv_tok; + uint32_t nbkv_head; + uint32_t batch_stride_q; + uint32_t batch_stride_k; + uint32_t batch_stride_v; + uint32_t batch_stride_m; + uint32_t batch_stride_o; + float softmax_scale; +}; + +struct vk_op_push_constants { + uint32_t KX; + uint32_t KY; + float param1; + float param2; + float param3; + float param4; +}; + +struct vk_wp_fused_expert_push_constants { + uint32_t expert_index; +}; + +// No `phase` field: phase A (gate/up -> hidden scratch) and phase B (down + +// weighted fold across experts) are now separate shaders/pipelines (see +// wp_fused_expert_batch_a.comp / wp_fused_expert_batch_b.comp) selected by +// which pipeline is dispatched, not by a runtime branch on a push constant. +// This struct is kept only as an unused placeholder so the generic +// ggml_vk_create_pipeline/ggml_vk_dispatch_pipeline plumbing (which threads a +// push_constant_size through every pipeline uniformly) needs no special case +// for a phase-less pipeline. +struct vk_wp_fused_expert_batch_push_constants { + uint32_t reserved; +}; + +struct vk_op_fwht_push_constants { + uint32_t n_rows; + uint32_t src_offset; + uint32_t dst_offset; + float scale; +}; + +struct vk_op_dsv4_hc_comb_push_constants { + uint32_t n_tokens; + + uint32_t nbm0; uint32_t nbm1; + uint32_t nbs0; + uint32_t nbb0; + uint32_t nbd0; uint32_t nbd1; uint32_t nbd2; + + uint32_t m_offset; + uint32_t s_offset; + uint32_t b_offset; + uint32_t d_offset; + + float eps; + uint32_t n_iter; +}; + +struct vk_op_dsv4_hc_pre_push_constants { + uint32_t n_embd; + uint32_t n_tokens; + + uint32_t nbx0; uint32_t nbx1; uint32_t nbx2; + uint32_t nbw0; uint32_t nbw1; uint32_t nbw2; + uint32_t nbd0; uint32_t nbd1; + + uint32_t x_offset; + uint32_t w_offset; + uint32_t d_offset; + + float scale; +}; + +struct vk_op_dsv4_hc_post_push_constants { + uint32_t n_embd; + uint32_t n_tokens; + + uint32_t nbx0; uint32_t nbx1; + uint32_t nbr0; uint32_t nbr1; uint32_t nbr2; + uint32_t nbp0; uint32_t nbp1; + uint32_t nbc0; uint32_t nbc1; uint32_t nbc2; + uint32_t nbd0; uint32_t nbd1; uint32_t nbd2; + + uint32_t x_offset; + uint32_t r_offset; + uint32_t p_offset; + uint32_t c_offset; + uint32_t d_offset; +}; + +struct vk_op_turbo_wht_push_constants { + uint32_t n_groups; // total groups = groups_per_head * n_heads + uint32_t group_size; // WHT group size (128, 64, or 32) + uint32_t direction; // 0 = forward, 1 = inverse + uint32_t has_scale; // 1 if InnerQ scale_inv buffer is valid, else 0 + uint32_t head_dim; // src->ne[0] + uint32_t groups_per_head; // head_dim / group_size + uint32_t src_offset; // misalignment offset (elements) for src buffer + uint32_t dst_offset; // misalignment offset (elements) for dst buffer + uint32_t scale_offset; // misalignment offset (elements) for scale buffer +}; + +struct vk_op_count_experts_push_constants { + uint32_t ne00; + uint32_t ne01; + uint32_t nb00; + uint32_t nb01; + uint32_t a_offset; + uint32_t n_experts; + uint32_t hoist_row_ids; + uint32_t ne00mp; + uint32_t ne00L; +}; + +struct vk_op_glu_push_constants { + uint32_t N; + uint32_t ne00; + uint32_t ne20; + uint32_t mode; // 0: default, 1: swapped, 2: split + float alpha; // for swiglu_oai + float limit; + uint32_t nb00; + uint32_t nb01; + uint32_t nb02; + uint32_t nb03; + uint32_t nb10; + uint32_t nb11; + uint32_t nb12; + uint32_t nb13; + uint32_t nb20; + uint32_t nb21; + uint32_t nb22; + uint32_t nb23; + uint32_t ne21; + uint32_t ne22; + uint32_t misalign_offsets; + uint32_t ne2_012mp; uint32_t ne2_012L; + uint32_t ne2_01mp; uint32_t ne2_01L; + uint32_t ne2_0mp; uint32_t ne2_0L; +}; + +static_assert(sizeof(vk_op_glu_push_constants) <= 128, "sizeof(vk_op_glu_push_constants) must be <= 128"); + +struct vk_op_unary_push_constants { + uint32_t ne; + uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03; + uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13; + uint32_t misalign_offsets; + float param1; float param2; float param3; float param4; + uint32_t ne0_012mp; uint32_t ne0_01mp; uint32_t ne0_0mp; uint32_t ne0_Ls; + uint32_t ne1_012mp; uint32_t ne1_01mp; uint32_t ne1_0mp; uint32_t ne1_Ls; +}; + +static_assert(sizeof(vk_op_unary_push_constants) <= 128, "sizeof(vk_op_unary_push_constants) must be <= 128"); + +static vk_op_unary_push_constants vk_op_unary_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst, int64_t ne = 0) { + GGML_ASSERT(ne != 0 || (ggml_nelements(src0) == ggml_nelements(dst))); + ne = ne != 0 ? ne : ggml_nelements(dst); + GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max()); + + vk_op_unary_push_constants p{}; + p.ne = (uint32_t)ne; + + size_t src0_tsize = ggml_type_size(src0->type); + p.ne00 = (uint32_t)src0->ne[0]; + p.ne01 = (uint32_t)src0->ne[1]; + p.ne02 = (uint32_t)src0->ne[2]; + p.ne03 = (uint32_t)src0->ne[3]; + p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize); + p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize); + p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize); + p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize); + + size_t dst_tsize = ggml_type_size(dst->type); + p.ne10 = (uint32_t)dst->ne[0]; + p.ne11 = (uint32_t)dst->ne[1]; + p.ne12 = (uint32_t)dst->ne[2]; + p.ne13 = (uint32_t)dst->ne[3]; + p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize); + p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize); + p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize); + p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize); + + return p; // offsets are initialized later in ggml_vk_op +} + +struct vk_op_pad_push_constants { + uint32_t ne; + uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03; + uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13; + uint32_t misalign_offsets; + uint32_t circular; + + uint32_t lp0; uint32_t rp0; + uint32_t lp1; uint32_t rp1; + uint32_t lp2; uint32_t rp2; + uint32_t lp3; uint32_t rp3; +}; + +static vk_op_pad_push_constants vk_op_pad_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst) { + int64_t ne = ggml_nelements(dst); + GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max()); + + vk_op_pad_push_constants p{}; + p.ne = (uint32_t)ne; + + size_t src0_tsize = ggml_type_size(src0->type); + p.ne00 = (uint32_t)src0->ne[0]; + p.ne01 = (uint32_t)src0->ne[1]; + p.ne02 = (uint32_t)src0->ne[2]; + p.ne03 = (uint32_t)src0->ne[3]; + p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize); + p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize); + p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize); + p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize); + + size_t dst_tsize = ggml_type_size(dst->type); + p.ne10 = (uint32_t)dst->ne[0]; + p.ne11 = (uint32_t)dst->ne[1]; + p.ne12 = (uint32_t)dst->ne[2]; + p.ne13 = (uint32_t)dst->ne[3]; + p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize); + p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize); + p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize); + p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize); + + p.lp0 = dst->op_params[0]; + p.rp0 = dst->op_params[1]; + p.lp1 = dst->op_params[2]; + p.rp1 = dst->op_params[3]; + p.lp2 = dst->op_params[4]; + p.rp2 = dst->op_params[5]; + p.lp3 = dst->op_params[6]; + p.rp3 = dst->op_params[7]; + p.circular = dst->op_params[8]; + + return p; // fastdiv values and offsets are initialized later in ggml_vk_op +} + +static void init_fastdiv_values(uint32_t d, uint32_t &mp, uint32_t &L) +{ + // compute L = ceil(log2(d)); + L = 0; + while (L < 32 && (uint32_t{1} << L) < d) { + L++; + } + + mp = (uint32_t)((uint64_t{1} << 32) * ((uint64_t{1} << L) - d) / d + 1); +} + +static uint32_t pack_fastdiv_L(uint32_t L0, uint32_t L1, uint32_t L2) { + return L0 | (L1 << 8) | (L2 << 16); +} + +template <typename T> void init_pushconst_fastdiv(T &p) { + GGML_UNUSED(p); + static_assert(!std::is_const<T>::value, "unexpected type"); +} + +template <> inline void init_pushconst_fastdiv(vk_op_unary_push_constants &p) { + // Compute magic values to divide by these six numbers. + uint32_t ne0_012L; + uint32_t ne0_01L; + uint32_t ne0_0L; + uint32_t ne1_012L; + uint32_t ne1_01L; + uint32_t ne1_0L; + + init_fastdiv_values(p.ne02*p.ne01*p.ne00, p.ne0_012mp, ne0_012L); + init_fastdiv_values(p.ne01*p.ne00, p.ne0_01mp, ne0_01L); + init_fastdiv_values(p.ne00, p.ne0_0mp, ne0_0L); + init_fastdiv_values(p.ne12*p.ne11*p.ne10, p.ne1_012mp, ne1_012L); + init_fastdiv_values(p.ne11*p.ne10, p.ne1_01mp, ne1_01L); + init_fastdiv_values(p.ne10, p.ne1_0mp, ne1_0L); + + p.ne0_Ls = pack_fastdiv_L(ne0_012L, ne0_01L, ne0_0L); + p.ne1_Ls = pack_fastdiv_L(ne1_012L, ne1_01L, ne1_0L); +} + +template <> inline void init_pushconst_fastdiv(vk_op_glu_push_constants &p) { + // GLU linearizes over dst, then uses dst coordinates for src0/src1. + init_fastdiv_values(p.ne22*p.ne21*p.ne20, p.ne2_012mp, p.ne2_012L); + init_fastdiv_values(p.ne21*p.ne20, p.ne2_01mp, p.ne2_01L); + init_fastdiv_values(p.ne20, p.ne2_0mp, p.ne2_0L); +} + +template <> inline void init_pushconst_fastdiv(vk_op_count_experts_push_constants &p) { + init_fastdiv_values(p.ne00, p.ne00mp, p.ne00L); +} + +struct vk_op_binary_push_constants { + uint32_t ne; + uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03; + uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13; + uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; uint32_t nb20; uint32_t nb21; uint32_t nb22; uint32_t nb23; + uint32_t misalign_offsets; + float param1; float param2; int32_t param3; +}; + +// SP2 paged attention push constants. Layouts must match +// paged_attn_scatter.comp / paged_attn.comp respectively. +struct vk_op_paged_scatter_pc { + uint32_t HS; + uint32_t BS; + uint32_t n_kv_heads; + uint32_t n_tokens; + // Outlier-matrix sweep (2026-07-01, investigative-only): mirrors + // GGML_TURBO4_64_OL_TABLE read on the host (see + // ggml_vk_turbo4_64_ol_use_n64_table below / ggml_cuda_turbo4_64_ol_use_n64_table + // in turbo-quant.cuh). Nonzero selects TURBO_CENTROIDS_4BIT_N64 / + // TURBO_MID_4BIT_N64 for the turbo4_64_ol/ol8/ol12 non-outlier elements + // instead of the default TURBO_CENTROIDS_4BIT/TURBO_MID_4BIT table. + // Ignored by all other cache types. Not a permanent field. + uint32_t use_n64_table; +}; + +struct vk_op_paged_attn_pc { + uint32_t HS; + uint32_t BS; + uint32_t n_heads; + uint32_t n_kv_heads; + uint32_t max_blocks_per_seq; + uint32_t n_seq; + uint32_t n_tokens; + float scale; + // See vk_op_paged_scatter_pc::use_n64_table above (investigative-only). + uint32_t use_n64_table; +}; + +// SP2 Task 5 split-K decode push constants. Must match +// paged_attn_decode.comp / paged_attn_decode_reduce.comp respectively. +struct vk_op_paged_decode_pc { + uint32_t HS; + uint32_t BS; + uint32_t n_heads; + uint32_t n_kv_heads; + uint32_t max_blocks_per_seq; + uint32_t n_seq; + uint32_t num_splits; + float scale; + // See vk_op_paged_scatter_pc::use_n64_table above (investigative-only). + uint32_t use_n64_table; +}; + +struct vk_op_paged_decode_reduce_pc { + uint32_t HS; + uint32_t n_heads; + uint32_t n_seq; + uint32_t num_splits; +}; +struct vk_op_concat_push_constants : vk_op_binary_push_constants {}; + +static_assert(sizeof(vk_op_concat_push_constants) == sizeof(vk_op_binary_push_constants)); + +static_assert(std::is_standard_layout_v<vk_op_concat_push_constants>); + +struct vk_op_multi_add_push_constants { + // shape for dst + uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; + + // strides for srcs+dst + uint32_t nb[MAX_PARAMETER_COUNT][4]; + + uint32_t rms_partials; +}; + +static_assert(MAX_PARAMETER_COUNT == 12); + +static_assert(sizeof(vk_op_multi_add_push_constants) <= 256); + +struct vk_op_topk_moe_push_constants { + uint32_t n_rows; + uint32_t n_experts_push; + uint32_t n_expert_used; + float clamp_min; + float clamp_max; + uint32_t gating_func; + uint32_t has_bias; + uint32_t with_norm; + float output_scale; + float output_bias; +}; + +struct vk_op_add_id_push_constants { + uint32_t ne0; + uint32_t ne1; + uint32_t s01; + uint32_t s02; + uint32_t s11; + uint32_t s21; +}; + +struct vk_op_diag_mask_push_constants { + uint32_t ncols; + uint32_t rows_per_channel; + int32_t n_past; +}; + +struct vk_op_rope_push_constants { + uint32_t rope_mode; + uint32_t nrows; + uint32_t n_dims; + uint32_t n_offs; + float freq_scale; + float freq_base; + float ext_factor; + float attn_factor; + float corr_dims[2]; + float theta_scale; + uint32_t has_ff; + int32_t sections[4]; + uint32_t is_imrope; + uint32_t is_back; + uint32_t set_rows_stride; + uint32_t ne00; + uint32_t ne01; + uint32_t ne02; + uint32_t nb01; + uint32_t nb02; + uint32_t nb03; + uint32_t nb11; + uint32_t nb12; + uint32_t nb13; + uint32_t a_offset; + uint32_t d_offset; +}; + +static_assert(sizeof(vk_op_rope_push_constants) <= 128, "sizeof(vk_op_rope_push_constants) must be <= 128"); + +struct vk_op_rms_norm_mul_rope_push_constants { + vk_op_binary_push_constants bin; + vk_op_rope_push_constants rope; +}; + +struct vk_op_soft_max_push_constants { + uint32_t KX; + uint32_t KY; + uint32_t ne00; + uint32_t ne01; + uint32_t ne02; + uint32_t ne12; + uint32_t ne13; + uint32_t nb11; + uint32_t nb12; + uint32_t nb13; + float scale; + float max_bias; + float m0; + float m1; + uint32_t n_head_log2; + uint32_t nrows_x; + uint32_t has_sinks; +}; + +struct vk_op_sinkhorn_norm_push_constants { + uint32_t n; + uint32_t nt; + float eps; + int32_t iters; + uint32_t a_offset; + uint32_t d_offset; +}; + +struct vk_op_argsort_push_constants { + uint32_t ncols; + uint32_t ncols_padded; + uint32_t ncols_padded_log2; + uint32_t nrows; + uint32_t order; + uint32_t outer_start; + uint32_t outer_end; + uint32_t inner_start; + uint32_t inner_end; +}; + +struct vk_op_topk_push_constants { + uint32_t orig_ncols; + uint32_t ncols_input; + uint32_t ncols_output; + uint32_t k; + uint32_t nrows; + uint32_t first_pass; + uint32_t last_pass; +}; + +struct vk_op_topk_radix_push_constants { + uint32_t ncols; + uint32_t k; + uint32_t nrows; + uint32_t n_tps; // QSA only + uint32_t n_blocks; // QSA only + uint32_t n_stream; // QSA only +}; + +struct vk_op_im2col_push_constants { + uint64_t dst_addr; + uint32_t batch_offset; uint32_t offset_delta; + uint32_t IC; + uint32_t IW; uint32_t IH; + uint32_t OW; uint32_t OH; + uint32_t KW; uint32_t KH; + uint32_t OH_batch; + uint32_t CHW; + int32_t s0; int32_t s1; + int32_t p0; int32_t p1; + int32_t d0; int32_t d1; + uint32_t batch_IC; +}; + +struct vk_op_im2col_3d_push_constants { + uint64_t dst_addr; + uint32_t nb10; + uint32_t nb11; + uint32_t nb12; + uint32_t nb13; + uint32_t s0; + uint32_t s1; + uint32_t s2; + uint32_t p0; + uint32_t p1; + uint32_t p2; + uint32_t d0; + uint32_t d1; + uint32_t d2; + uint32_t IW; + uint32_t IH; + uint32_t ID; + uint32_t IC; + uint32_t KW; + uint32_t OH; + uint32_t KD_KH_KW; + uint32_t KH_KW; + uint32_t IC_KD_KH_KW; + uint32_t N_OD_OH; + uint32_t OD_OH; + uint32_t OD_OH_OW_IC_KD_KH_KW; + uint32_t OH_OW_IC_KD_KH_KW; + uint32_t OW_IC_KD_KH_KW; + uint32_t misalign_offsets; +}; + +struct vk_op_timestep_embedding_push_constants { + uint32_t nb1; + uint32_t dim; + uint32_t max_period; +}; + +struct vk_op_col2im_1d_push_constants { + uint32_t T_out; + uint32_t OC; + uint32_t K_OC; + uint32_t T_in; + uint32_t K; + int32_t stride; + int32_t p0; +}; + +struct vk_op_conv_transpose_1d_push_constants { + uint32_t Cout; + uint32_t Cin; + uint32_t K; + uint32_t L; + uint32_t KL; + + uint32_t nb01; + uint32_t nb02; + uint32_t nb11; + uint32_t nb1; + + int32_t s0; +}; + +struct vk_op_snake_push_constants { + uint32_t ne0; + uint32_t ne1; +}; + +struct vk_op_pool1d_push_constants { + uint32_t IL; + uint32_t OL; + uint32_t OC; + uint32_t pelements; + uint32_t op; + int32_t k0; + int32_t s0; + int32_t p0; +}; + +struct vk_op_pool2d_push_constants { + uint32_t IW; uint32_t IH; + uint32_t OW; uint32_t OH; + uint32_t OC; + uint32_t pelements; + uint32_t op; + int32_t k0; int32_t k1; + int32_t s0; int32_t s1; + int32_t p0; int32_t p1; +}; + +struct vk_op_rwkv_wkv6_push_constants { + uint32_t B; + uint32_t T; + uint32_t C; + uint32_t H; +}; + +struct vk_op_rwkv_wkv7_push_constants { + uint32_t B; + uint32_t T; + uint32_t C; + uint32_t H; +}; + +struct vk_op_gated_linear_attn_push_constants { + uint32_t B; + uint32_t T; + uint32_t C; + uint32_t H; + float scale; +}; + +struct vk_op_lightning_indexer_push_constants { + uint32_t n_kv; + uint32_t n_heads; + uint32_t n_tokens; + uint32_t n_streams; + uint32_t n_masks; + uint32_t dispatch_x; + uint32_t q_nb1; + uint32_t q_nb2; + uint32_t q_nb3; + uint32_t k_nb2; + uint32_t k_nb3; + uint32_t w_nb1; + uint32_t w_nb3; + uint32_t m_nb1; + uint32_t m_nb3; + uint32_t d_nb1; + uint32_t d_nb3; +}; + +static_assert(sizeof(vk_op_lightning_indexer_push_constants) <= 128); + +struct vk_op_gated_delta_net_push_constants { + uint32_t H; + uint32_t n_tokens; + uint32_t n_seqs; + uint32_t s_off; + uint32_t sq1, sq2, sq3; + uint32_t sv1, sv2, sv3; + uint32_t sb1, sb2, sb3; + uint32_t neq1, rq3; + float scale; + uint32_t K; +}; + +struct vk_op_ssm_scan_push_constants { + uint32_t nb02, nb03, nb12, nb13; + uint32_t nb21, nb22, nb31; + uint32_t nb42, nb43, nb52, nb53; + uint32_t s_off; + uint32_t n_head, d_head, n_group, n_tok; + uint32_t n_seq, K; +}; + +struct vk_op_ssm_conv_push_constants { + uint32_t nb01, nb02; + uint32_t nb11; + uint32_t dst_nb0, dst_nb1, dst_nb2; + uint32_t nc, ncs, nr, n_t, n_s; +}; + +struct vk_op_conv2d_push_constants { + uint32_t Cout; + uint32_t Cin; + uint32_t N; + + uint32_t W; + uint32_t H; + uint32_t OW; + uint32_t OH; + + uint32_t nb01; + uint32_t nb02; + uint32_t nb03; + + uint32_t nb11; + uint32_t nb12; + uint32_t nb13; + + uint32_t nb1; + uint32_t nb2; + uint32_t nb3; + + // init_fastdiv_values constants for dividing by OW, OW*OH + uint32_t OWmp; uint32_t OWL; + uint32_t OWOHmp; uint32_t OWOHL; +}; + +template <> inline void init_pushconst_fastdiv(vk_op_conv2d_push_constants &p) { + // Compute magic values to divide by OW, OW*OH + init_fastdiv_values(p.OW, p.OWmp, p.OWL); + init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL); +} + +struct vk_op_conv3d_push_constants { + uint32_t OC; + uint32_t IC; + uint32_t N; + + uint32_t IW; + uint32_t IH; + uint32_t ID; + uint32_t OW; + uint32_t OH; + uint32_t OD; + + uint32_t nb01; + uint32_t nb02; + uint32_t nb03; + + uint32_t nb11; + uint32_t nb12; + uint32_t nb13; + + uint32_t nb1; + uint32_t nb2; + uint32_t nb3; + + uint32_t OWmp; uint32_t OWL; + uint32_t OWOHmp; uint32_t OWOHL; + uint32_t OWOHODmp; uint32_t OWOHODL; +}; + +template <> inline void init_pushconst_fastdiv(vk_op_conv3d_push_constants &p) { + init_fastdiv_values(p.OW, p.OWmp, p.OWL); + init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL); + init_fastdiv_values(p.OW*p.OH*p.OD, p.OWOHODmp, p.OWOHODL); +} + +struct vk_op_conv2d_dw_push_constants { + uint32_t ne; + uint32_t batches; + uint32_t channels; + uint32_t dst_w; + uint32_t dst_h; + uint32_t src_w; + uint32_t src_h; + uint32_t knl_w; + uint32_t knl_h; + int32_t stride_x; + int32_t stride_y; + int32_t pad_x; + int32_t pad_y; + int32_t dilation_x; + int32_t dilation_y; +}; + +struct vk_op_upscale_push_constants { + uint32_t ne; uint32_t a_offset; uint32_t d_offset; + uint32_t ne00; uint32_t ne01; + uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03; + uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; + float sf0; float sf1; float sf2; float sf3; + float pixel_offset; +}; + +struct vk_op_sum_rows_push_constants +{ + uint32_t n_cols; + uint32_t ne01, ne02; + uint32_t nb01, nb02, nb03; + uint32_t nb11, nb12, nb13; + float weight; + uint32_t misalign_offsets; + uint32_t ne0_12mp, ne0_12L; + uint32_t ne0_1mp, ne0_1L; +}; + +static vk_op_sum_rows_push_constants vk_op_sum_rows_push_constants_init(const ggml_tensor * src, const ggml_tensor * dst, int64_t n_cols) { + uint32_t type_size = (uint32_t)ggml_type_size(src->type); + vk_op_sum_rows_push_constants p = {}; + p.n_cols = (uint32_t)n_cols; + p.ne01 = (uint32_t)src->ne[1]; + p.ne02 = (uint32_t)src->ne[2]; + p.nb01 = (uint32_t)src->nb[1] / type_size; + p.nb02 = (uint32_t)src->nb[2] / type_size; + p.nb03 = (uint32_t)src->nb[3] / type_size; + p.nb11 = (uint32_t)dst->nb[1] / type_size; + p.nb12 = (uint32_t)dst->nb[2] / type_size; + p.nb13 = (uint32_t)dst->nb[3] / type_size; + p.weight = 1.0f; + return p; +} + +template <> inline void init_pushconst_fastdiv(vk_op_sum_rows_push_constants &p) { + init_fastdiv_values(p.ne01*p.ne02, p.ne0_12mp, p.ne0_12L); + init_fastdiv_values(p.ne01, p.ne0_1mp, p.ne0_1L); +} + +struct vk_quantize_q8_1_push_constants { + uint32_t ne; + uint32_t num_blocks; +}; + +struct vk_op_flash_attn_split_k_reduce_push_constants { + uint32_t D; + uint32_t ne1; + uint32_t ne2; + uint32_t ne3; + uint32_t k_num; + uint32_t sinks; +}; + +struct vk_op_flash_attn_mask_opt_push_constants { + uint32_t nem0; + uint32_t nem1; + uint32_t nem2; + uint32_t nbm1; + uint32_t nbm2; + uint32_t nbm3; + uint32_t nbd1; + uint32_t nbd2; + uint32_t nbd3; +}; + +struct vk_op_flash_attn_sparse_compact_push_constants { + uint32_t KV; + uint32_t nem1; + uint32_t nem2; + uint32_t nbm1; + uint32_t nbm2; + uint32_t nbm3; + uint32_t n_kv_max; +}; + +template <typename T> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, T &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + GGML_UNUSED(p); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(src2); + GGML_UNUSED(src3); + GGML_UNUSED(dst); + static_assert(!std::is_const<T>::value, "unexpected type"); + GGML_ASSERT(!src0 || get_misalign_bytes(ctx, src0) == 0); + GGML_ASSERT(!src1 || get_misalign_bytes(ctx, src1) == 0); + GGML_ASSERT(!src2 || get_misalign_bytes(ctx, src2) == 0); + GGML_ASSERT(!src3 || get_misalign_bytes(ctx, src3) == 0); + GGML_ASSERT(!dst || get_misalign_bytes(ctx, dst) == 0); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_p021_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); + const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + p.b_offset = b_offset; + p.d_offset = d_offset; + + GGML_UNUSED(src0); + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_nc_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); + const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + p.b_offset = b_offset; + p.d_offset = d_offset; + + GGML_UNUSED(src0); + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_fwht_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + p.src_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + p.dst_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + GGML_UNUSED(src1); + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_turbo_wht_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + p.src_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + p.dst_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + p.scale_offset = src1 ? get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type) : 0u; + + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_comb_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + p.m_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + p.s_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); + p.b_offset = get_misalign_bytes(ctx, src2) / ggml_type_size(src2->type); + p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_pre_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + p.x_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + p.w_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); + p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_post_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + p.x_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + p.r_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); + p.p_offset = get_misalign_bytes(ctx, src2) / ggml_type_size(src2->type); + p.c_offset = src3 ? get_misalign_bytes(ctx, src3) / ggml_type_size(src3->type) : 0; + p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); +} + +template <typename T> size_t push_constant_size(const T &t) { + static_assert(std::is_class<T>::value, "T must be a struct/class"); + GGML_UNUSED(t); + return sizeof(T); +} + +template <typename T> size_t push_constant_size(const std::vector<T> &t) { + GGML_UNUSED(t); + return sizeof(T) * t.size(); +} + +template <typename T, uint32_t N> size_t push_constant_size(const std::array<T, N> &t) { + GGML_UNUSED(t); + return sizeof(T) * N; +} + +template <typename T> const T *push_constant_data(const T &t) { + static_assert(std::is_class<T>::value, "T must be a struct/class"); + return &t; +} + +template <typename T> const T *push_constant_data(const std::vector<T> &t) { + return t.data(); +} + +template <typename T, uint32_t N> const T *push_constant_data(const std::array<T, N> &t) { + return t.data(); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_unary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + p.misalign_offsets = (a_offset << 16) | d_offset; + + GGML_UNUSED(src1); + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_sinkhorn_norm_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + p.a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + GGML_UNUSED(src1); + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_glu_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + const uint32_t b_offset = src1 ? get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type) : a_offset; + const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + GGML_ASSERT(a_offset < (1u << 8)); + GGML_ASSERT(b_offset < (1u << 8)); + GGML_ASSERT(d_offset < (1u << 8)); + + p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset; + + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_sum_rows_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + p.misalign_offsets = (a_offset << 16) | d_offset; + + GGML_UNUSED(src1); + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_pad_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + p.misalign_offsets = (a_offset << 16) | d_offset; + + GGML_UNUSED(src1); + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_im2col_3d_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + const uint32_t a_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); + const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + p.misalign_offsets = (a_offset << 16) | d_offset; + + GGML_UNUSED(src0); + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_binary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); + const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + GGML_ASSERT(a_offset <= 0xFFFF); + GGML_ASSERT(b_offset <= 0xFF); + GGML_ASSERT(d_offset <= 0xFF); + + p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset; + + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_concat_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type); + const uint32_t a_offset = get_misalign_bytes(ctx, src0) / unit_size; + const uint32_t b_offset = get_misalign_bytes(ctx, src1) / unit_size; + const uint32_t d_offset = get_misalign_bytes(ctx, dst) / unit_size; + + p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset; + + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_upscale_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + p.a_offset = a_offset; + p.d_offset = d_offset; + + GGML_UNUSED(src1); + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_rope_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + p.a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); + p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + + GGML_UNUSED(src1); + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + +static vk_op_binary_push_constants ggml_vk_rms_norm_push_constants( + const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst, + float eps, uint32_t num_partials) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); + + return { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + eps, 0.0f, (int32_t)num_partials, + }; +} + diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h new file mode 100644 index 000000000000..c1056f1d0261 --- /dev/null +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -0,0 +1,1569 @@ +#pragma once + +#include "ggml-vulkan.h" + +#include <vulkan/vulkan_core.h> + +#include <chrono> +#if defined(GGML_VULKAN_RUN_TESTS) || defined(GGML_VULKAN_CHECK_RESULTS) +#include "ggml-cpu.h" +#endif + +#define VULKAN_HPP_DISPATCH_LOADER_DYNAMIC 1 + +#if VK_HEADER_VERSION >= 301 +namespace vk::detail { class DispatchLoaderDynamic; } +using vk::detail::DispatchLoaderDynamic; +#else +namespace vk { class DispatchLoaderDynamic; } +using vk::DispatchLoaderDynamic; +#endif + +DispatchLoaderDynamic & ggml_vk_default_dispatcher(); + +#define VULKAN_HPP_DEFAULT_DISPATCHER ggml_vk_default_dispatcher() + +#include <vulkan/vulkan.hpp> + +#ifndef VK_NV_cooperative_matrix_decode_vector +#define VK_NV_cooperative_matrix_decode_vector 1 +#define VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME "VK_NV_cooperative_matrix_decode_vector" +#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV ((VkStructureType)1000689000) +typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV { + VkStructureType sType; + void* pNext; + VkBool32 cooperativeMatrixDecodeVector; +} VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV; +#endif + +#if __has_include(<spirv/unified1/spirv.hpp>) +# include <spirv/unified1/spirv.hpp> +#elif __has_include(<spirv-headers/spirv.hpp>) +# include <spirv-headers/spirv.hpp> +#elif __has_include(<spirv.hpp>) +# include <spirv.hpp> +#else + // Fallback to let the compiler throw a standard "file not found" error +# include <spirv/unified1/spirv.hpp> +#endif + +#include <algorithm> +#include <atomic> + +#include <cmath> + +#include <iomanip> + +#include <iostream> + +#include <tuple> + +#include <vector> + +#include <deque> + +#include <sstream> + +#include <utility> + +#include <memory> + +#include <limits> + +#include <map> + +#include <set> + +#include <unordered_map> + +#include <shared_mutex> + +#include <mutex> + +#include <future> + +#include <condition_variable> + +#include <thread> + +#if defined(_MSC_VER) +# define NOMINMAX 1 +# include <windows.h> +# define YIELD() YieldProcessor() +#elif defined(__clang__) || defined(__GNUC__) +# if defined(__x86_64__) ||defined(__i386__) +# include <immintrin.h> +# define YIELD() _mm_pause() +# elif defined(__arm__) || defined(__aarch64__) +# if defined(__clang__) +# include <arm_acle.h> +# define YIELD() __yield() +# else +# define YIELD() asm volatile("yield") +# endif +# endif +#endif + +#if !defined(YIELD) +#define YIELD() +#endif + +#include "ggml-impl.h" + +#include "ggml-backend-impl.h" + +#include "ggml-vulkan-shaders.hpp" + +#if !defined(VK_KHR_shader_bfloat16) + +#define VK_KHR_shader_bfloat16 1 +#define VK_KHR_SHADER_BFLOAT16_SPEC_VERSION 1 +#define VK_KHR_SHADER_BFLOAT16_EXTENSION_NAME "VK_KHR_shader_bfloat16" +#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR ((VkStructureType)1000141000) +#define VK_COMPONENT_TYPE_BFLOAT16_KHR ((VkComponentTypeKHR)1000141000) + +typedef struct VkPhysicalDeviceShaderBfloat16FeaturesKHR { + VkStructureType sType; + void* pNext; + VkBool32 shaderBFloat16Type; + VkBool32 shaderBFloat16DotProduct; + VkBool32 shaderBFloat16CooperativeMatrix; +} VkPhysicalDeviceShaderBfloat16FeaturesKHR; +#endif + +#if !defined(VK_VALVE_shader_mixed_float_dot_product) +#define VK_VALVE_shader_mixed_float_dot_product 1 +#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_SPEC_VERSION 1 +#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_EXTENSION_NAME "VK_VALVE_shader_mixed_float_dot_product" +#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE ((VkStructureType)1000673000) +typedef struct VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE { + VkStructureType sType; + void* pNext; + VkBool32 shaderMixedFloatDotProductFloat16AccFloat32; + VkBool32 shaderMixedFloatDotProductFloat16AccFloat16; + VkBool32 shaderMixedFloatDotProductBFloat16Acc; + VkBool32 shaderMixedFloatDotProductFloat8AccFloat32; +} VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE; +#endif + +#if !defined(VK_EXT_shader_ocp_microscaling_types) +#define VK_EXT_shader_ocp_microscaling_types 1 +#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_SPEC_VERSION 1 +#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME "VK_EXT_shader_ocp_microscaling_types" +#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OCP_MICROSCALING_TYPES_FEATURES_EXT ((VkStructureType)1000672000) +typedef struct VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT { + VkStructureType sType; + void* pNext; + VkBool32 shaderFloat4; + VkBool32 shaderFloat6; + VkBool32 shaderFloat8UnsignedE8M0; + VkBool32 shaderMXInt8; +} VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT; +#endif + +#if !defined(VK_EXT_shader_float8) +#define VK_EXT_shader_float8 1 +#define VK_EXT_SHADER_FLOAT8_SPEC_VERSION 1 +#define VK_EXT_SHADER_FLOAT8_EXTENSION_NAME "VK_EXT_shader_float8" +#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT ((VkStructureType)1000567000) +typedef struct VkPhysicalDeviceShaderFloat8FeaturesEXT { + VkStructureType sType; + void* pNext; + VkBool32 shaderFloat8; + VkBool32 shaderFloat8CooperativeMatrix; +} VkPhysicalDeviceShaderFloat8FeaturesEXT; +#endif + +#ifndef VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME +#define VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME "VK_KHR_internally_synchronized_queues" +#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INTERNALLY_SYNCHRONIZED_QUEUES_FEATURES_KHR ((VkStructureType)1000504000) +#define VK_DEVICE_QUEUE_CREATE_INTERNALLY_SYNCHRONIZED_BIT_KHR ((VkDeviceQueueCreateFlagBits)0x00000004) + +// Compile-time constant guaranteed; no runtime initialization overhead +static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR = + static_cast<vk::DeviceQueueCreateFlagBits>(0x00000004); + +typedef struct VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR { + VkStructureType sType; + void* pNext; + VkBool32 internallySynchronizedQueues; +} VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR; +#else +static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR = vk::DeviceQueueCreateFlagBits::eInternallySynchronizedKHR; +#endif + +#define ROUNDUP_POW2(M, N) (((M) + (N) - 1) & ~((N) - 1)) + +#define CEIL_DIV(M, N) (((M) / (N)) + (((M) % (N)) != 0)) + +static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; } + +#define VK_VENDOR_ID_AMD 0x1002 + +#define VK_VENDOR_ID_APPLE 0x106b + +#define VK_VENDOR_ID_INTEL 0x8086 + +#define VK_VENDOR_ID_NVIDIA 0x10de + +#define VK_VENDOR_ID_QUALCOMM 0x5143 + +#define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256 + +#define VK_CHECK(err, msg, dev) \ + do { \ + vk::Result err_; \ + try { \ + err_ = (err); \ + } catch (vk::DeviceLostError &) { \ + ggml_vk_print_device_lost_info(dev); \ + GGML_LOG_ERROR("ggml_vulkan: %s at %s:%d\n", \ + #err, __FILE__, __LINE__); \ + throw; \ + } \ + if (err_ != vk::Result::eSuccess) { \ + GGML_LOG_ERROR("ggml_vulkan: %s error %s at %s:%d\n", \ + #err, to_string(err_).c_str(), __FILE__, __LINE__); \ + throw vk::SystemError(vk::make_error_code(err_), \ + "ggml_vulkan: " msg); \ + } \ + } while (0) + +#ifdef GGML_VULKAN_DEBUG +#define VK_LOG_DEBUG(msg) std::cerr << msg << std::endl +#else +#define VK_LOG_DEBUG(msg) ((void) 0) +#endif // GGML_VULKAN_DEBUG + +#define MAX_PARAMETER_COUNT 12 + +#define MAX_FUSED_ADDS (MAX_PARAMETER_COUNT - 3) + +struct vk_pipeline_struct; + +typedef std::shared_ptr<struct vk_pipeline_struct> vk_pipeline; + +struct vk_pipeline_struct { + std::string name; + vk::ShaderModule shader_module; + vk::PipelineLayout layout; + vk::Pipeline pipeline; + uint32_t push_constant_size; + uint32_t parameter_count; + std::array<uint32_t, 3> wg_denoms; + uint32_t align; + // true if fields have been set by ggml_vk_create_pipeline + bool initialized {}; + // true while a compile is in flight, used to dedupe concurrent claims. + // Protected by device->compile_mutex. + bool compile_pending {}; + // set to true when the shader has been compiled + std::atomic<bool> compiled {}; + // number of registers used, extracted from pipeline executable properties + uint32_t register_count {}; + +#if defined(VK_EXT_shader_64bit_indexing) + bool is_64b_indexing {}; +#endif + // linked list of pipelines for multiple compilation variants. + // currently only used to compile a 64-bit indexing variant. + vk_pipeline next; +}; + +typedef std::weak_ptr<vk_pipeline_struct> vk_pipeline_ref; + +struct vk_matmul_pipeline_key { + ggml_type type_a; + ggml_type type_b; + bool mul_mat_id; + bool f16acc; + + bool operator<(const vk_matmul_pipeline_key & o) const { + return std::tie(type_a, type_b, mul_mat_id, f16acc) + < std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc); + } +}; + +struct vk_matmul_pipeline_pair { + vk_pipeline unaligned; + vk_pipeline aligned; + uint32_t align; +}; + +struct vk_tile_config { + std::vector<uint32_t> warptile; + std::array<uint32_t, 3> wg_denoms; + uint32_t align; +}; + +using matmul_tile_selector_t = std::function<uint32_t( + uint32_t m, uint32_t n, uint32_t k, uint32_t shader_core_count, + const std::vector<vk_matmul_pipeline_pair>& configs)>; + +struct vk_device_struct; + +typedef std::shared_ptr<vk_device_struct> vk_device; + +typedef std::weak_ptr<vk_device_struct> vk_device_ref; + +struct vk_buffer_struct; + +typedef std::shared_ptr<vk_buffer_struct> vk_buffer; + +typedef std::weak_ptr<vk_buffer_struct> vk_buffer_ref; + +struct ggml_backend_vk_buffer_type_context { + std::string name; + vk_device device; +}; + +struct vk_command_buffer { + vk::CommandBuffer buf; + uint64_t use_counter = 0; + bool in_use = false; +}; + +struct vk_queue; + +struct vk_command_pool { + void init(vk_device& device, vk_queue *q_); + void destroy(vk::Device& device); + + vk::CommandPool pool; + // Using deque so the pointers to command buffers + // remain valid even if we add more + std::deque<vk_command_buffer> cmd_buffers; + + vk_queue *q; + + size_t buffers_in_use() const { + return std::count_if(cmd_buffers.begin(), cmd_buffers.end(), + [](const auto& cb) { return cb.in_use; }); + } +}; + +struct vk_queue_handle { + vk::Queue queue; + vk_device_ref device; + std::mutex * device_submit_mutex = nullptr; + // *** QUEUE QUIESCENCE EPOCHS (bookkeeping only -- inert on its own). *** + // + // submit_epoch counts every vkQueueSubmit ever issued through this handle; + // it is bumped BEFORE the submit so an observer never sees the two counters + // equal while a batch is in flight. waited_epoch is advanced (monotonically) + // to the submit_epoch value captured just before a fence wait that covers + // everything submitted to this queue so far. submit_epoch == waited_epoch + // therefore means "this queue has no work outstanding that anyone still has + // to wait for", which is what lets WP_VK_HOST_READ_FASTPATH read + // host-coherent memory with a plain memcpy instead of a barrier submit. + // + // Missing a publish point only makes the predicate pessimistic (the fast + // path stops engaging); missing a BUMP would make it unsound, which is why + // the bump lives in the submit overrides -- the single choke point for + // every vkQueueSubmit in this file -- and not at the call sites. + std::atomic<uint64_t> submit_epoch {0}; + std::atomic<uint64_t> waited_epoch {0}; + virtual void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) = 0; + virtual void lock() {} // no-op by default (internally synchronized case) + virtual void unlock() {} + virtual ~vk_queue_handle() = default; +}; + +// True when nothing submitted to this queue is still awaiting a wait. +static inline bool ggml_vk_queue_quiesced(const vk_queue_handle & h) { + return h.waited_epoch.load(std::memory_order_acquire) == + h.submit_epoch.load(std::memory_order_acquire); +} + +// Capture the epoch to publish once a covering fence wait returns. +static inline uint64_t ggml_vk_queue_epoch(const vk_queue_handle & h) { + return h.submit_epoch.load(std::memory_order_acquire); +} + +// Publish (monotonically) that everything up to `epoch` has been waited on. +static inline void ggml_vk_queue_mark_waited(vk_queue_handle & h, uint64_t epoch) { + uint64_t cur = h.waited_epoch.load(std::memory_order_relaxed); + while (cur < epoch && + !h.waited_epoch.compare_exchange_weak(cur, epoch, + std::memory_order_release, + std::memory_order_relaxed)) { + } +} + +struct vk_queue_handle_synchronized : vk_queue_handle { + std::mutex mutex; + void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override; + + void lock() override { mutex.lock(); } + void unlock() override { mutex.unlock(); } +}; + +struct vk_queue_handle_unsynchronized : vk_queue_handle { + void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override; + + // lock()/unlock() inherited no-ops +}; + +struct vk_queue { + uint32_t queue_family_index; + std::shared_ptr<vk_queue_handle> handle; + + vk_command_pool cmd_pool; + + vk::PipelineStageFlags stage_flags; + + bool transfer_only; +}; + +// Upper bound on the batch width the mat-vec (GEMV) path can serve, and hence +// the size of the per-NUM_COLS pipeline arrays. The *effective* cap is per +// device (vk_device::mul_mat_vec_max_cols): only architectures that have been +// measured to stay below the mul_mm path at these widths go above 8, so nobody +// else pays the pipeline-compile cost for the extra instantiations. +static constexpr uint32_t mul_mat_vec_max_cols = 16; +static constexpr uint32_t mul_mat_vec_max_cols_default = 8; + +static constexpr uint32_t p021_max_gqa_ratio = 8; + +enum vk_device_architecture { + OTHER, + AMD_GCN, + AMD_RDNA1, + AMD_RDNA2, + AMD_RDNA3, + INTEL_XE1, + INTEL_XE2, + NVIDIA_PRE_TURING, + NVIDIA_TURING, +}; + +enum vk_conv_shapes { + CONV_SHAPE_128x128, + CONV_SHAPE_64x32, + CONV_SHAPE_32x256, + CONV_SHAPE_64x128, + CONV_SHAPE_COUNT, +}; + +struct vk_conv_block_size { + uint32_t K; + uint32_t NPQ; + uint32_t CRS; +}; + +inline vk_conv_block_size vk_conv_block_sizes[CONV_SHAPE_COUNT] = { + // K NPQ CRS + { 128, 128, 16 }, // CONV_SHAPE_128x128 + { 64, 32, 32 }, // CONV_SHAPE_64x32 + { 32, 256, 16 }, // CONV_SHAPE_32x256 + { 64, 128, 16 }, // CONV_SHAPE_64x128 +}; + +enum dmmv_wg_sizes { + DMMV_WG_SIZE_SUBGROUP, + DMMV_WG_SIZE_LARGE, + DMMV_WG_SIZE_COUNT, +}; + +enum FaCodePath { + FA_SCALAR, + FA_COOPMAT1, + FA_COOPMAT2, +}; + +struct vk_fa_pipeline_state { + uint32_t HSK, HSV; + uint32_t Br, Bc; + uint32_t D_split, row_split; + bool shmem_staging; + FaCodePath path; + uint32_t workgroup_size, subgroup_size; + bool aligned; + bool f32acc; + uint32_t flags; + uint32_t limit_occupancy_shmem; + ggml_type k_type; + ggml_type v_type; + + bool operator<(const vk_fa_pipeline_state &b) const { + return std::tie(HSK, HSV, Br, Bc, D_split, row_split, shmem_staging, path, workgroup_size, subgroup_size, aligned, f32acc, flags, limit_occupancy_shmem, k_type, v_type) < + std::tie(b.HSK, b.HSV, b.Br, b.Bc, b.D_split, b.row_split, b.shmem_staging, b.path, b.workgroup_size, b.subgroup_size, b.aligned, b.f32acc, b.flags, b.limit_occupancy_shmem, b.k_type, b.v_type); + } +}; + +struct vk_conv2d_pipeline_state { + vk_conv2d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t p0, uint32_t p1, uint32_t d0, uint32_t d1, uint32_t KW, uint32_t KH, uint32_t aligned) + : s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), KW(KW), KH(KH), aligned(aligned) {} + + uint32_t s0, s1, p0, p1, d0, d1, KW, KH; + // when set, shader can skip K/CRS/NPQ bounds checks and address clamps + uint32_t aligned; + + bool operator<(const vk_conv2d_pipeline_state &b) const { + return std::tie(s0, s1, p0, p1, d0, d1, KW, KH, aligned) < + std::tie(b.s0, b.s1, b.p0, b.p1, b.d0, b.d1, b.KW, b.KH, b.aligned); + } +}; + +struct vk_conv3d_pipeline_state { + vk_conv3d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t s2, uint32_t p0, uint32_t p1, uint32_t p2, + uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned) + : s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned) {} + + uint32_t s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD; + uint32_t aligned; + + bool operator<(const vk_conv3d_pipeline_state &b) const { + return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned) < + std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned); + } +}; + +struct vk_solve_tri_pipeline_state { + vk_solve_tri_pipeline_state(uint32_t N, uint32_t K) + : N(N), K(K) {} + + uint32_t N, K; + + bool operator<(const vk_solve_tri_pipeline_state &b) const { + return std::tie(N, K) < + std::tie(b.N, b.K); + } +}; + +enum shader_reduction_mode { + SHADER_REDUCTION_MODE_SHMEM, + SHADER_REDUCTION_MODE_HYBRID, + SHADER_REDUCTION_MODE_SUBGROUP, + SHADER_REDUCTION_MODE_COUNT, +}; + +static constexpr uint32_t num_argsort_pipelines = 11; + +static constexpr uint32_t num_topk_moe_pipelines = 10; + +static constexpr uint32_t num_topk_pipelines = 11; + +static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax_norm{ GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, + GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE, + GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, + GGML_OP_RESHAPE }; + +static constexpr std::initializer_list<ggml_op> topk_moe_sigmoid_norm_bias{ GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, + GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, + GGML_OP_DIV, GGML_OP_RESHAPE }; + +static constexpr std::initializer_list<ggml_op> topk_moe_sqrt_softplus_norm_bias{ GGML_OP_UNARY, GGML_OP_SQRT, + GGML_OP_RESHAPE, GGML_OP_ADD, + GGML_OP_ARGSORT, GGML_OP_VIEW, + GGML_OP_GET_ROWS, GGML_OP_RESHAPE, + GGML_OP_SUM_ROWS, GGML_OP_CLAMP, + GGML_OP_DIV, GGML_OP_RESHAPE }; + +static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, + GGML_OP_VIEW, GGML_OP_GET_ROWS }; + +static constexpr std::initializer_list<ggml_op> topk_moe_late_softmax { GGML_OP_ARGSORT, GGML_OP_VIEW, + GGML_OP_GET_ROWS, GGML_OP_RESHAPE, + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE }; + +static constexpr std::initializer_list<ggml_op> snake_pattern { GGML_OP_MUL, GGML_OP_SIN, + GGML_OP_SQR, GGML_OP_MUL, + GGML_OP_ADD }; + +static constexpr std::initializer_list<ggml_op> topk_qsa_pattern { GGML_OP_GET_ROWS, GGML_OP_PERMUTE, + GGML_OP_CONT, GGML_OP_CPY, + GGML_OP_RESHAPE, GGML_OP_ADD, + GGML_OP_TOP_K }; + +static constexpr std::initializer_list<std::array<int, 3>> topk_qsa_edges { + { 1, 0, 0 }, // permute->src[0] == get_rows + { 2, 0, 1 }, // cont->src[0] == permute + { 4, 0, 3 }, // reshape->src[0] == cpy (mask cast) + { 5, 0, 2 }, // add->src[0] == cont + { 5, 1, 4 }, // add->src[1] == reshape + { 6, 0, 5 }, // top_k->src[0] == add +}; + +static constexpr std::initializer_list<ggml_op> rms_norm_mul_add_mul_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_MUL }; + +static constexpr std::initializer_list<ggml_op> rms_norm_mul_add_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }; + +static constexpr std::initializer_list<ggml_op> rms_norm_mul_rope_view_set_rows_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }; + +static constexpr std::initializer_list<ggml_op> rms_norm_view_set_rows_pattern { GGML_OP_RMS_NORM, GGML_OP_VIEW, GGML_OP_SET_ROWS }; + +static constexpr std::initializer_list<ggml_op> rope_view_set_rows_pattern { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }; + +static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_norm_edges { + { 1, 0, 0 }, // reshape->src[0] == softmax + { 2, 0, 0 }, // argsort->src[0] == softmax + { 3, 0, 2 }, // view->src[0] == argsort + { 4, 0, 1 }, // get_rows->src[0] == reshape + { 4, 1, 3 }, // get_rows->src[1] == view + { 5, 0, 4 }, // reshape->src[0] == get_rows + { 6, 0, 5 }, // sum_rows->src[0] == reshape + { 7, 0, 6 }, // clamp->src[0] == sum_rows + { 8, 0, 5 }, // div->src[0] == reshape + { 8, 1, 7 }, // div->src[1] == clamp + { 9, 0, 8 }, // reshape->src[0] == div +}; + +static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sigmoid_norm_bias_edges { + { 1, 0, 0 }, // reshape->src[0] == sigmoid + { 2, 0, 0 }, // add->src[0] == sigmoid + { 3, 0, 2 }, // argsort->src[0] == add + { 4, 0, 3 }, // view->src[0] == argsort + { 5, 0, 1 }, // get_rows->src[0] == reshape + { 5, 1, 4 }, // get_rows->src[1] == view + { 6, 0, 5 }, // reshape->src[0] == get_rows + { 7, 0, 6 }, // sum_rows->src[0] == reshape + { 8, 0, 7 }, // clamp->src[0] == sum_rows + { 9, 0, 6 }, // div->src[0] == reshape + { 9, 1, 8 }, // div->src[1] == clamp + {10, 0, 9 }, // reshape->src[0] == div +}; + +static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sqrt_softplus_norm_bias_edges { + { 1, 0, 0 }, // sqrt->src[0] == softplus + { 2, 0, 1 }, // reshape->src[0] == sqrt + { 3, 0, 1 }, // add->src[0] == sqrt + { 4, 0, 3 }, // argsort->src[0] == add + { 5, 0, 4 }, // view->src[0] == argsort + { 6, 0, 2 }, // get_rows->src[0] == reshape + { 6, 1, 5 }, // get_rows->src[1] == view + { 7, 0, 6 }, // reshape->src[0] == get_rows + { 8, 0, 7 }, // sum_rows->src[0] == reshape + { 9, 0, 8 }, // clamp->src[0] == sum_rows + {10, 0, 7 }, // div->src[0] == reshape + {10, 1, 9 }, // div->src[1] == clamp + {11, 0,10 }, // reshape->src[0] == div +}; + +static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_edges { + { 1, 0, 0 }, // reshape->src[0] == softmax + { 2, 0, 0 }, // argsort->src[0] == softmax + { 3, 0, 2 }, // view->src[0] == argsort + { 4, 0, 1 }, // get_rows->src[0] == reshape + { 4, 1, 3 }, // get_rows->src[1] == view +}; + +static constexpr std::initializer_list<std::array<int, 3>> topk_moe_late_softmax_edges { + { 1, 0, 0 }, // view->src[0] == argsort + { 2, 1, 1 }, // get_rows->src[1] == view + { 3, 0, 2 }, // reshape->src[0] == get_rows + { 4, 0, 3 }, // soft_max->src[0] == reshape + { 5, 0, 4 }, // reshape->src[0] == soft_max +}; + +enum topk_moe_mode { + TOPK_MOE_EARLY_SOFTMAX, + TOPK_MOE_EARLY_SOFTMAX_NORM, + TOPK_MOE_LATE_SOFTMAX, + TOPK_MOE_SIGMOID_NORM_BIAS, + TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS, + TOPK_MOE_COUNT, +}; + +enum rms_norm_mode { + RMS_NORM_MUL, + RMS_NORM_MUL_ADD, + RMS_NORM_MUL_ADD_MUL, + RMS_NORM_MUL_ROPE, + RMS_NORM_MUL_ROPE_VIEW_SET_ROWS, + RMS_NORM_VIEW_SET_ROWS, + RMS_NORM_COUNT, +}; + +static constexpr std::initializer_list<std::array<int, 3>> rope_view_set_rows_edges { + { 1, 0, 0 }, // view->src[0] == rope + { 2, 0, 1 }, // set_rows->src[0] == view +}; + +static constexpr std::initializer_list<std::array<int, 3>> rms_norm_mul_rope_view_set_rows_edges { + { 1, 0, 0 }, // mul->src[0] == rms + { 2, 0, 1 }, // rope->src[0] == mul + { 3, 0, 2 }, // view->src[0] == rope + { 4, 0, 3 }, // set_rows->src[0] == view +}; + +static constexpr std::initializer_list<std::array<int, 3>> rms_norm_view_set_rows_edges { + { 1, 0, 0 }, // view->src[0] == rms_norm + { 2, 0, 1 }, // set_rows->src[0] == view +}; + +static constexpr std::array<ggml_type, 9> lightning_indexer_k_types = { + GGML_TYPE_F32, + GGML_TYPE_F16, + GGML_TYPE_BF16, + GGML_TYPE_Q8_0, + GGML_TYPE_Q5_1, + GGML_TYPE_Q5_0, + GGML_TYPE_Q4_1, + GGML_TYPE_Q4_0, + GGML_TYPE_IQ4_NL, +}; + +class vk_memory_logger; + +struct vk_device_struct { + std::recursive_mutex mutex; + std::mutex queue_submit_mutex; + mutable std::shared_mutex pinned_memory_mutex; + + // Guards compile_pending, all_pipelines, and the dynamic pipeline maps + // (flash_attn, fa_mask_opt, solve_tri, conv2d, etc). The actual compile + // runs with no lock held, so different pipelines can compile in parallel. + // Lock order is device->mutex -> compile_mutex, never the reverse. + std::mutex compile_mutex; + std::condition_variable compile_cv; + + uint32_t debug_cmdbuf_idx {}; + + vk::PhysicalDevice physical_device; + vk::PhysicalDeviceProperties properties; + std::string name; + uint64_t max_memory_allocation_size; + uint64_t max_buffer_size; + uint64_t suballocation_block_size; + uint64_t min_imported_host_pointer_alignment; + bool external_memory_host {}; + // VK_KHR_external_semaphore_fd: lets two vk::Devices in this process share + // one timeline semaphore payload, which the tensor-parallel AllReduce + // (ggml-vulkan-allreduce.hpp) uses for cross-device ordering. + bool external_semaphore_fd {}; + bool fp16; + bool bf16; + bool pipeline_robustness; + bool memory_priority; + vk::Device device; + uint32_t vendor_id; + vk::DriverId driver_id; + vk_device_architecture architecture; + std::unique_ptr<vk_queue> compute_queue; + std::unique_ptr<vk_queue> transfer_queue; + bool single_queue; + bool support_async; + bool async_use_transfer_queue; + bool has_internally_synchronized_queues = false; + uint32_t subgroup_size; + uint32_t subgroup_size_log2; + uint32_t shader_core_count; + // Effective GEMV batch-width cap for this device, <= mul_mat_vec_max_cols. + // Set in ggml_vk_load_shaders before any mat-vec pipeline is created. + uint32_t mul_mat_vec_max_cols_eff = mul_mat_vec_max_cols_default; + bool uma; + bool prefer_host_memory; + bool float_controls_rte_fp16; + bool float_controls_denorm_preserve_fp16; + bool subgroup_basic; + bool subgroup_arithmetic; + bool subgroup_shuffle; + bool subgroup_ballot; + bool subgroup_clustered; + bool subgroup_vote; + bool multi_add; + bool shader_int64; + bool buffer_device_address; + bool vulkan_memory_model; + + bool add_rms_fusion; + uint32_t partials_binding_alignment; + uint32_t max_nodes_per_submit; + + bool shader_64b_indexing; + + bool integer_dot_product; + // 0: default, 1: force mmvq, -1: disable mmvq + int32_t mmvq_mode; + + bool subgroup_size_control; + uint32_t subgroup_min_size; + uint32_t subgroup_max_size; + bool subgroup_require_full_support; + + // floor(log2(maxComputeWorkGroupInvocations)) + uint32_t max_workgroup_size_log2 {}; + + bool coopmat_support; + bool coopmat_acc_f32_support {}; + bool coopmat_acc_f16_support {}; + bool coopmat_bf16_support {}; + bool coopmat_support_16x16x16_f16acc {}; + bool coopmat_support_16x16x16_f32acc {}; + bool coopmat1_fa_support {}; + uint32_t coopmat_m; + uint32_t coopmat_n; + uint32_t coopmat_k; + + bool coopmat_int_support; + uint32_t coopmat_int_m; + uint32_t coopmat_int_n; + uint32_t coopmat_int_k; + + bool coopmat2; + bool coopmat2_bf16_support {}; + bool coopmat2_decode_vector; + + bool dot2_f16 {}; + bool ocp_fp4 {}; + + bool pipeline_executable_properties_support {}; + + bool device_fault {}; + PFN_vkGetDeviceFaultInfoEXT pfn_vkGetDeviceFaultInfoEXT {}; + + bool serialize_submissions {}; + + const ggml_cgraph * diag_cgraph {}; + int diag_prev_start = -1; + int diag_prev_end = -1; + + size_t idx; + + bool mul_mat_l[GGML_TYPE_COUNT]; + bool mul_mat_m[GGML_TYPE_COUNT]; + bool mul_mat_s[GGML_TYPE_COUNT]; + bool mul_mat_id_l[GGML_TYPE_COUNT]; + bool mul_mat_id_m[GGML_TYPE_COUNT]; + bool mul_mat_id_s[GGML_TYPE_COUNT]; + + // Separate flags for the q8_1 (integer dot) mmq path, whose shader uses + // a different shared-memory layout than the float matmul shaders. + bool mul_mat_l_int[GGML_TYPE_COUNT]; + bool mul_mat_m_int[GGML_TYPE_COUNT]; + bool mul_mat_s_int[GGML_TYPE_COUNT]; + bool mul_mat_id_l_int[GGML_TYPE_COUNT]; + bool mul_mat_id_m_int[GGML_TYPE_COUNT]; + bool mul_mat_id_s_int[GGML_TYPE_COUNT]; + + vk::DescriptorSetLayout dsl; + + std::map<vk_matmul_pipeline_key, std::vector<vk_matmul_pipeline_pair>> pipeline_matmul; + matmul_tile_selector_t matmul_tile_selector; + matmul_tile_selector_t matmul_id_tile_selector; + + vk_pipeline pipeline_matmul_split_k_reduce; + vk_pipeline pipeline_quantize_q8_1_x4; + + vk_pipeline pipeline_dequant[GGML_TYPE_COUNT]; + vk_pipeline pipeline_dequant_transpose[GGML_TYPE_COUNT]; // fused dequant+transpose for FA quant-KV + vk_pipeline pipeline_dequant_mul_mat_vec_f32_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols]; + vk_pipeline pipeline_dequant_mul_mat_vec_f16_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols]; + vk_pipeline pipeline_dequant_mul_mat_vec_id_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT]; + + vk_pipeline pipeline_dequant_mul_mat_vec_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols]; + vk_pipeline pipeline_dequant_mul_mat_vec_id_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT]; + + vk_pipeline pipeline_wp_fused_expert_q5_1; + vk_pipeline pipeline_wp_fused_expert_q8_0; + // Batched fast path (see ggml_backend_vk_wp_fused_expert): TWO shaders, + // one dispatch each, covering every selected expert of one request + // instead of one dispatch per expert. Deliberately two separate + // pipelines/shader modules (not one shader branching on a `phase` push + // constant, as an earlier version did) so phase B contains no + // barrier()/shared-memory staging that depended on every invocation of + // the workgroup -- including masked-out ones -- reaching it; that + // design hung the RX 480 (RADV/ACO) even after replacing an + // early-return-before-barrier with a clamp. Phase A never touches the + // down buffer, so it has no down-type variant; phase B does. + vk_pipeline pipeline_wp_fused_expert_batch_a; + vk_pipeline pipeline_wp_fused_expert_batch_b_q5_1; + vk_pipeline pipeline_wp_fused_expert_batch_b_q8_0; + + vk_pipeline pipeline_mul_mat_vec_p021_f16_f32[p021_max_gqa_ratio]; + vk_pipeline pipeline_mul_mat_vec_nc_f16_f32; + vk_pipeline pipeline_get_rows[GGML_TYPE_COUNT]; + vk_pipeline pipeline_get_rows_f32[GGML_TYPE_COUNT]; + vk_pipeline pipeline_get_rows_back_f32; + vk_pipeline pipeline_acc_f32; + vk_pipeline pipeline_set_f32; + + // [src0 0=fp32,1=fp16][src1 0=fp32,1=fp16][dst 0=fp32,1=fp16] + vk_pipeline pipeline_add[2][2][2]; + vk_pipeline pipeline_add_norepeat[2][2][2]; + vk_pipeline pipeline_sub[2][2][2]; + vk_pipeline pipeline_sub_norepeat[2][2][2]; + vk_pipeline pipeline_mul[2][2][2]; + vk_pipeline pipeline_mul_norepeat[2][2][2]; + vk_pipeline pipeline_div[2][2][2]; + vk_pipeline pipeline_div_norepeat[2][2][2]; + vk_pipeline pipeline_add_rms[2][2][2]; + vk_pipeline pipeline_add_rms_norepeat[2][2][2]; + + // indexed by num_additional_fused_ops == num_adds - 1 + vk_pipeline pipeline_multi_add[MAX_FUSED_ADDS]; + vk_pipeline pipeline_multi_add_rms[MAX_FUSED_ADDS]; + + vk_pipeline pipeline_add_id_f32; + + vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64; + vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32; + vk_pipeline pipeline_scale_f32; + vk_pipeline pipeline_log[2]; + vk_pipeline pipeline_tri[2]; + vk_pipeline pipeline_diag[2]; + vk_pipeline pipeline_clamp[2]; + vk_pipeline pipeline_pad_f32; + vk_pipeline pipeline_pad_reflect_1d_f32; + vk_pipeline pipeline_roll_f32; + vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32; + vk_pipeline pipeline_repeat_i16; + vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16, pipeline_cpy_f16_f32, pipeline_cpy_f32_bf16, pipeline_cpy_bf16_f32, pipeline_cpy_f32_i32, pipeline_cpy_i32_f32; + vk_pipeline pipeline_contig_cpy_f32_f32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_f16_f16, pipeline_contig_cpy_f16_f32, pipeline_contig_cpy_f32_bf16, pipeline_contig_cpy_bf16_f32, pipeline_contig_cpy_f32_i32, pipeline_contig_cpy_i32_f32; + vk_pipeline pipeline_cpy_f32_quant[GGML_TYPE_COUNT]; + vk_pipeline pipeline_cpy_quant_f32[GGML_TYPE_COUNT]; + vk_pipeline pipeline_cpy_transpose_16, pipeline_cpy_transpose_32; + vk_pipeline pipeline_cpy_transpose_02_16, pipeline_cpy_transpose_02_32; + // [src0 0=fp32,1=fp16][dst] + vk_pipeline pipeline_set_rows_i32[2][GGML_TYPE_COUNT]; + vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT]; + // SP2 paged attention (GGML_OP_PAGED_ATTN_MT), indexed by KV cache type. + vk_pipeline pipeline_paged_attn_scatter[GGML_TYPE_COUNT]; + vk_pipeline pipeline_paged_attn[GGML_TYPE_COUNT]; + // SP2 Task 5: split-K decode (q_len==1). Pass-1 per cache type, pass-2 reduce type-agnostic. + vk_pipeline pipeline_paged_attn_decode[GGML_TYPE_COUNT]; + vk_pipeline pipeline_paged_attn_decode_reduce; + vk_pipeline pipeline_norm_f32; + vk_pipeline pipeline_group_norm_f32; + vk_pipeline pipeline_turbo_wht_f32; + vk_pipeline pipeline_rms_norm_f32; + vk_pipeline pipeline_rms_norm_mul_f32; + vk_pipeline pipeline_rms_norm_mul_add_f32; + vk_pipeline pipeline_rms_norm_mul_add_mul_f32; + vk_pipeline pipeline_rms_norm_mul_add_partials_f32; + vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32; + vk_pipeline pipeline_rms_norm_set_rows_f32_f32; + vk_pipeline pipeline_rms_norm_set_rows_f32_f16; + vk_pipeline pipeline_rms_norm_partials_f32; + vk_pipeline pipeline_rms_norm_mul_partials_f32; + vk_pipeline pipeline_rms_norm_mul_rope_f32_f32; + vk_pipeline pipeline_rms_norm_mul_rope_f32_f16; + vk_pipeline pipeline_rms_norm_back_f32; + vk_pipeline pipeline_l2_norm_f32; + vk_pipeline pipeline_sinkhorn_norm_f32[3]; + + // [src/dst 0=fp32,1=fp16] + vk_pipeline pipeline_exp[2]; + vk_pipeline pipeline_expm1[2]; + vk_pipeline pipeline_elu[2]; + vk_pipeline pipeline_gelu[2]; + vk_pipeline pipeline_gelu_erf[2]; + vk_pipeline pipeline_gelu_quick[2]; + vk_pipeline pipeline_silu[2]; + vk_pipeline pipeline_relu[2]; + vk_pipeline pipeline_sqr[2]; + vk_pipeline pipeline_sqrt[2]; + vk_pipeline pipeline_sin[2]; + vk_pipeline pipeline_cos[2]; + vk_pipeline pipeline_xielu[2]; + vk_pipeline pipeline_neg[2]; + vk_pipeline pipeline_tanh[2]; + vk_pipeline pipeline_sigmoid[2]; + vk_pipeline pipeline_hardsigmoid[2]; + vk_pipeline pipeline_hardswish[2]; + vk_pipeline pipeline_abs[2]; + vk_pipeline pipeline_softplus[2]; + vk_pipeline pipeline_step[2]; + vk_pipeline pipeline_round[2]; + vk_pipeline pipeline_ceil[2]; + vk_pipeline pipeline_floor[2]; + vk_pipeline pipeline_trunc[2]; + vk_pipeline pipeline_sgn[2]; + + // fused UNARY+MUL pipelines: [op][f16][norepeat][op_on_b] + vk_pipeline pipeline_unary_mul[4][2][2][2]; + + vk_pipeline pipeline_add1_f16_f16; + vk_pipeline pipeline_add1_f16_f32; + vk_pipeline pipeline_add1_f32_f32; + + vk_pipeline pipeline_arange_f32; + + vk_pipeline pipeline_fill_f32; + vk_pipeline pipeline_fill_f16; + + vk_pipeline pipeline_geglu[2]; + vk_pipeline pipeline_reglu[2]; + vk_pipeline pipeline_swiglu[2]; + vk_pipeline pipeline_swiglu_oai[2]; + vk_pipeline pipeline_swiglu_clamp[2]; + vk_pipeline pipeline_geglu_erf[2]; + vk_pipeline pipeline_geglu_quick[2]; + + vk_pipeline pipeline_leaky_relu[2]; + vk_pipeline pipeline_silu_back_f32; + vk_pipeline pipeline_diag_mask_inf_f32; + vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16; + vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512; + vk_pipeline pipeline_soft_max_back_f32; + + vk_pipeline pipeline_soft_max_large1_f32, pipeline_soft_max_large1_f32_f16; + vk_pipeline pipeline_soft_max_large2_f32, pipeline_soft_max_large2_f32_f16; + vk_pipeline pipeline_soft_max_large3_f32, pipeline_soft_max_large3_f32_f16; + + vk_pipeline pipeline_rope_norm_f32, pipeline_rope_norm_f16, pipeline_rope_norm_f32_f16; + vk_pipeline pipeline_rope_neox_f32, pipeline_rope_neox_f16, pipeline_rope_neox_f32_f16; + vk_pipeline pipeline_rope_multi_f32, pipeline_rope_multi_f16, pipeline_rope_multi_f32_f16; + vk_pipeline pipeline_rope_vision_f32, pipeline_rope_vision_f16; + vk_pipeline pipeline_argsort_f32[num_argsort_pipelines]; + vk_pipeline pipeline_argsort_large_f32[num_argsort_pipelines]; + vk_pipeline pipeline_topk_f32[num_topk_pipelines]; + vk_pipeline pipeline_topk_radix_f32; + vk_pipeline pipeline_topk_radix_qsa; // qwen4 QSA indexer fusion (f16 mask) + vk_pipeline pipeline_sum_rows_f32; + vk_pipeline pipeline_cross_entropy_loss_f32, pipeline_cross_entropy_loss_f32_wg512; + vk_pipeline pipeline_cross_entropy_loss_back_f32, pipeline_cross_entropy_loss_back_f32_wg512; + vk_pipeline pipeline_fwht_f32[4]; + vk_pipeline pipeline_cumsum_f32; + vk_pipeline pipeline_cumsum_small_f32; + vk_pipeline pipeline_cumsum_multipass1_f32; + vk_pipeline pipeline_cumsum_multipass2_f32; + vk_pipeline pipeline_argmax_f32; + vk_pipeline pipeline_count_equal_i32; + vk_pipeline pipeline_dsv4_hc_comb_f32; + vk_pipeline pipeline_dsv4_hc_pre_f32; + vk_pipeline pipeline_dsv4_hc_pre_gated_f32; + vk_pipeline pipeline_dsv4_hc_post_f32; + vk_pipeline pipeline_dsv4_hc_post_nocomb_f32; + std::map<vk_solve_tri_pipeline_state, vk_pipeline> pipeline_solve_tri_f32; + vk_pipeline pipeline_im2col_f32, pipeline_im2col_f32_f16; + vk_pipeline pipeline_im2col_3d_f32, pipeline_im2col_3d_f32_f16; + vk_pipeline pipeline_timestep_embedding_f32; + vk_pipeline pipeline_conv_transpose_1d_f32; + vk_pipeline pipeline_col2im_1d_f32; + vk_pipeline pipeline_col2im_1d_f16; + vk_pipeline pipeline_col2im_1d_bf16; + vk_pipeline pipeline_out_prod_f32; + vk_pipeline pipeline_snake_f32; + vk_pipeline pipeline_snake_f16; + vk_pipeline pipeline_snake_bf16; + vk_pipeline pipeline_pool1d_f32; + vk_pipeline pipeline_pool2d_f32; + vk_pipeline pipeline_rwkv_wkv6_f32; + vk_pipeline pipeline_rwkv_wkv7_f32; + vk_pipeline pipeline_gated_linear_attn_f32; + vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT]; + // [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128 + vk_pipeline pipeline_gated_delta_net[4][2]; + // chunked (UT-transform) variant, [size_idx]; GDA only, null when unsupported + vk_pipeline pipeline_gated_delta_net_chunked[4]; + // columns covered by one chunked workgroup, [size_idx] (0 when the pipeline is null) + uint32_t gated_delta_net_chunked_cols[4] = {0, 0, 0, 0}; + vk_pipeline pipeline_ssm_scan_f32_d128; + vk_pipeline pipeline_ssm_scan_f32_d256; + vk_pipeline pipeline_ssm_conv_f32; + vk_pipeline pipeline_ssm_conv_silu_f32; + vk_pipeline pipeline_ssm_conv_bias_silu_f32; + vk_pipeline pipeline_opt_step_adamw_f32; + vk_pipeline pipeline_opt_step_sgd_f32; + std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f32[CONV_SHAPE_COUNT]; + std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f16_f32[CONV_SHAPE_COUNT]; + std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f32[CONV_SHAPE_COUNT]; + std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f16_f32[CONV_SHAPE_COUNT]; + std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f32[CONV_SHAPE_COUNT]; + std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f16_f32[CONV_SHAPE_COUNT]; + vk_pipeline pipeline_conv2d_dw_whcn_f32, pipeline_conv2d_dw_whcn_f16_f32; + vk_pipeline pipeline_conv2d_dw_cwhn_f32, pipeline_conv2d_dw_cwhn_f16_f32; + + std::map<vk_fa_pipeline_state, vk_pipeline> pipeline_flash_attn_f32_f16; + + std::map<std::pair<uint32_t, uint32_t>, vk_pipeline> pipeline_fa_mask_opt; + + vk_pipeline pipeline_fa_sparse_compact; + vk_pipeline pipeline_fa_sparse_compact_subgroup; + bool fa_sparse_compact_use_subgroups; + + vk_pipeline pipeline_flash_attn_split_k_reduce; + std::map<std::tuple<uint32_t, uint32_t, uint32_t, uint32_t>, std::pair<vk_pipeline, vk_pipeline>> pipeline_xe_fa_decode_dual_phases; + vk_pipeline pipeline_count_experts; + + // [2] is for whether to take n_experts from spec constant (0) or push constant (1) + vk_pipeline pipeline_topk_moe[num_topk_moe_pipelines][2]; + + std::vector<vk_pipeline_ref> all_pipelines; + + std::vector<std::tuple<void*, size_t, vk_buffer>> pinned_memory; + + vk::Fence fence; + vk_buffer sync_staging; + vk_buffer host_read_staging; + vk_buffer host_read_source; + size_t host_read_source_offset {}; + size_t host_read_source_size {}; + // Device-local scratch for the fused-expert batch fast path's Phase A + // output (silu(gate)*up per expert/token/hidden-unit), consumed by + // Phase B in the same command buffer. Grows on demand, never shrinks -- + // mirrors host_read_staging's ensure_*() pattern below. + vk_buffer wp_fused_batch_scratch; + + ggml_backend_buffer_type buffer_type; + + bool disable_fusion; + bool disable_host_visible_vidmem; + bool allow_sysmem_fallback; + bool disable_graph_optimize; + + std::unique_ptr<vk_memory_logger> memory_logger; + + ~vk_device_struct(); + +}; + +inline void vk_command_pool::init(vk_device& device, vk_queue *q_) { + cmd_buffers.clear(); + q = q_; + + vk::CommandPoolCreateInfo command_pool_create_info( + vk::CommandPoolCreateFlags(VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT), + q->queue_family_index); + pool = device->device.createCommandPool(command_pool_create_info); +} + +inline void vk_command_pool::destroy(vk::Device& device) { + device.destroyCommandPool(pool); + pool = nullptr; + cmd_buffers.clear(); +} + +struct vk_buffer_struct { + vk::Buffer buffer = VK_NULL_HANDLE; + vk::DeviceMemory device_memory = VK_NULL_HANDLE; + vk::MemoryPropertyFlags memory_property_flags; + void * ptr; + size_t size = 0; + vk::DeviceAddress bda_addr {}; + + vk_device device; + + ~vk_buffer_struct() { + if (size == 0) { + return; + } + VK_LOG_DEBUG("~vk_buffer_struct(" << buffer << ", " << size << ")"); + + device->device.freeMemory(device_memory); + device->device.destroyBuffer(buffer); + } +}; + +struct vk_subbuffer { + vk_buffer buffer; + uint64_t offset; + uint64_t size; + const ggml_tensor * tensor = nullptr; + bool allow_misalign = false; + + operator vk::DescriptorBufferInfo() const { + return { buffer->buffer, offset, size }; + } +}; + +struct vk_semaphore { + vk::Semaphore s; + uint64_t value; +}; + +struct vk_event { + std::vector<vk::Event> events_free; // Events available for reuse + std::vector<vk::Event> events_submitted; // Events that are fully submitted and can be reused on next synchronize + vk::Event event; + bool has_event; + + vk_semaphore tl_semaphore; + vk_command_buffer* cmd_buffer = nullptr; + uint64_t cmd_buffer_use_counter = 0; +}; + +struct vk_submission { + vk_command_buffer* buffer = nullptr; + std::vector<vk_semaphore> wait_semaphores; + std::vector<vk_semaphore> signal_semaphores; +}; + +typedef std::vector<vk_submission> vk_sequence; + +#define MAT_VEC_FUSION_FLAGS_BIAS0 0x1 + +#define MAT_VEC_FUSION_FLAGS_BIAS1 0x2 + +#define MAT_VEC_FUSION_FLAGS_SCALE0 0x4 + +#define MAT_VEC_FUSION_FLAGS_SCALE1 0x8 + +struct vk_staging_memcpy { + vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {} + + void * dst; + const void * src; + size_t n; +}; + +struct vk_staging_memset { + vk_staging_memset(void * _dst, uint32_t _val, size_t _n) : dst(_dst), val(_val), n(_n) {} + + void * dst; + uint32_t val; + size_t n; +}; + +struct vk_context_struct { + vk_submission * s; + vk_command_buffer * submitted_cmd_buf = nullptr; + std::vector<vk_sequence> seqs; + + int exit_tensor_idx; + + std::vector<vk_staging_memcpy> in_memcpys; + std::vector<vk_staging_memcpy> out_memcpys; + std::vector<vk_staging_memset> memsets; + + std::vector<std::string> debug_labels; + + // Pending contiguous same-type CPY regions, flushed as one vkCmdCopyBuffer. + vk_buffer cpy_src; + vk_buffer cpy_dst; + std::vector<vk::BufferCopy> cpy_regions; + + vk_command_pool * p {}; +}; + +typedef std::shared_ptr<vk_context_struct> vk_context; + +typedef std::weak_ptr<vk_context_struct> vk_context_ref; + +struct ggml_backend_vk_context; + +struct ggml_backend_vk_graph_plan { + struct descriptor_update { + vk::DescriptorSet descriptor_set; + std::vector<vk_subbuffer> sources; + }; + + ggml_backend_vk_context * owner = nullptr; + vk_command_pool command_pool; + std::vector<vk_context> contexts; + std::vector<vk::DescriptorPool> descriptor_pools; + std::vector<vk::DescriptorSet> descriptor_sets; + std::vector<descriptor_update> descriptor_updates; + uint32_t descriptor_set_idx = 0; + uint32_t pipeline_descriptor_set_requirements = 0; + bool unsupported = false; +}; + +struct ggml_vk_garbage_collector { + std::vector<vk_semaphore> tl_semaphores; + std::vector<vk_semaphore> semaphores; + std::vector<vk::Event> events; + std::vector<vk_context> contexts; +}; + +#define VK_LOG_MEMORY(msg) if (vk_memory_logger_enabled) { std::cerr << "ggml_vulkan memory: " << msg << std::endl; } + +static std::string format_size(size_t size) { + const size_t kib = 1024; + const size_t mib = kib * 1024; + const size_t gib = mib * 1024; + + std::ostringstream oss; + oss << std::fixed << std::setprecision(2); + + if (size >= gib) { + oss << static_cast<double>(size) / gib << " GiB"; + } else if (size >= mib) { + oss << static_cast<double>(size) / mib << " MiB"; + } else if (size >= kib) { + oss << static_cast<double>(size) / kib << " KiB"; + } else { + oss << size << " B"; + } + + return oss.str(); +} + +class vk_memory_logger { +public: + vk_memory_logger(): total_device(0), total_host(0) {} + void log_allocation(vk_buffer_ref buf_ref, size_t size); + void log_deallocation(vk_buffer_ref buf_ref); + +private: + std::map<vk::Buffer, size_t> allocations; // Track allocations + size_t total_device; + size_t total_host; + static std::mutex log_mutex; +}; + +inline std::mutex vk_memory_logger::log_mutex; + +class vk_perf_logger { + public: + void print_timings(bool force = false); + + + std::string get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops); + + + void log_timing(const ggml_tensor * node, const char *fusion_name, uint64_t time) { + uint64_t n_flops; + std::string name = get_node_fusion_name(node, fusion_name, &n_flops); + if (n_flops) { + flops[name].push_back(n_flops); + } + timings[name].push_back(time); + } + + void log_timing(const std::vector<ggml_tensor *> &nodes, const std::vector<const char *> &names, uint64_t time) { + uint64_t total_flops = 0; + std::string name; + for (size_t n = 0; n < nodes.size(); ++n) { + uint64_t n_flops = 0; + name += get_node_fusion_name(nodes[n], names[n], &n_flops); + total_flops += n_flops; + + if (n != nodes.size() - 1) { + name += ", "; + } + } + if (total_flops) { + flops[name].push_back(total_flops); + } + timings[name].push_back(time); + } + + private: + std::map<std::string, std::vector<uint64_t>> timings; + std::map<std::string, std::vector<uint64_t>> flops; + uint32_t print_count {}; +}; + +struct ggml_backend_vk_context { + std::string name; + + vk_device device; + + size_t semaphore_idx, event_idx; + ggml_vk_garbage_collector gc; + size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k, prealloc_size_add_rms_partials, prealloc_size_add_rms_partials_offset; + vk_buffer prealloc_x, prealloc_y, prealloc_split_k, prealloc_add_rms_partials, sync_staging; + // Per-expert block offsets for weight paging. One slice per mul_mat_id node + // because dispatches execute after recording; cursor resets each graph. + vk_buffer prealloc_wp_expert_off; + size_t wp_expert_off_cursor = 0; + std::vector<vk_buffer> wp_expert_off_retired; + vk::Fence fence, almost_ready_fence; + bool submit_pending {}; + bool almost_ready_fence_pending {}; + // WP_VK_FENCE_ON_LAST_SUBMIT: true when ctx->fence has a pending signal that + // was attached to the graph's own final submit rather than to a trailing + // fence-only vkQueueSubmit. Invariant: while it is true, NOTHING has been + // submitted on this context since -- every submitter drains it first. + bool fence_submitted {}; + // Compute-queue submit_epoch as of the submission ctx->fence rides on, so a + // later drain publishes coverage for exactly that much and no more. + uint64_t fence_epoch {}; + // Set before op_add and unset after op_rms_norm to indicate that the add should + // write partial sums to accumulate the square of the vector components + bool do_add_rms_partials_offset_calculation; + bool do_add_rms_partials; + + uint64_t last_total_flops {UINT64_MAX}; + + // Cache most recent tensor that was converted into prealloc_y, and what pipeline it used to convert. + vk_pipeline_struct * prealloc_y_last_pipeline_used {}; + const ggml_tensor * prealloc_y_last_tensor_used {}; + // True when the K dimension in prealloc_y is padded. + bool prealloc_y_last_k_padded {}; + + // Track which nodes have been used since the last sync, and whether they were written to + std::vector<const ggml_tensor *> unsynced_nodes_written; + std::vector<const ggml_tensor *> unsynced_nodes_read; + // Track which prealloc buffers have pending reads that need to be synchronized. + // These are checked before writing to the buffer (and call ggml_vk_sync_buffers if set), + // and set to true after the buffer contents are consumed. + bool prealloc_x_need_sync, prealloc_y_need_sync, prealloc_split_k_need_sync; + + vk_context_ref compute_ctx; + + // Non-null only while a persistent graph plan is being recorded. + ggml_backend_vk_graph_plan * recording_plan = nullptr; + + vk_context_ref transfer_ctx; + vk_semaphore transfer_semaphore; + uint64_t transfer_semaphore_last_submitted {}; + + std::vector<vk_context_ref> tensor_ctxs; + + std::vector<vk::DescriptorPool> descriptor_pools; + std::vector<vk::DescriptorSet> descriptor_sets; + std::array<vk::DescriptorBufferInfo, MAX_PARAMETER_COUNT> descriptor_buffer_infos{}; + uint32_t descriptor_set_idx {}; + uint32_t pipeline_descriptor_set_requirements {}; + + vk_command_pool compute_cmd_pool; + vk_command_pool transfer_cmd_pool; + + // number of additional consecutive nodes that are being fused with the + // node currently being processed + int num_additional_fused_ops {}; + // Bitmask of which fused ops need to write an intermediate value to memory. + // Bit 'i' means nodes[start_of_fusion + i] writes to memory. + // If there's no fusion, bit 0 is still set. + int fused_ops_write_mask {}; + topk_moe_mode fused_topk_moe_mode {}; + bool fused_topk_moe_scale {}; + // QSA indexer gather+add+top_k fused into one radix-select + bool fused_topk_qsa {}; + rms_norm_mode fused_rms_norm_mode {RMS_NORM_COUNT}; + + // for GGML_VK_PERF_LOGGER and WP_VK_NODE_TS + std::unique_ptr<vk_perf_logger> perf_logger; + vk::QueryPool query_pool; + std::vector<const char *> query_fusion_names; + std::vector<int> query_fusion_node_count; + std::vector<ggml_tensor *> query_nodes; + std::vector<int> query_node_idx; + std::vector<ggml_op> query_ops; + int32_t num_queries {}; + int32_t query_idx {}; + bool node_ts_pending {}; + ggml_backend_vk_node_ts node_ts {}; +}; + +struct ggml_backend_vk_buffer_context { + vk_device_ref device; + vk_buffer dev_buffer; + std::string name; + + ggml_backend_vk_buffer_context(vk_device_ref device, vk_buffer&& dev_buffer, std::string& name) : + device(device), + dev_buffer(dev_buffer), + name(name) { + } + + ~ggml_backend_vk_buffer_context(); + +}; + +struct vk_instance_t { + vk::Instance instance; + + bool debug_utils_support = false; // VK_EXT_debug_utils enabled + PFN_vkSetDebugUtilsObjectNameEXT pfn_vkSetDebugUtilsObjectNameEXT = {}; + PFN_vkQueueBeginDebugUtilsLabelEXT pfn_vkQueueBeginDebugUtilsLabelEXT = {}; + PFN_vkQueueEndDebugUtilsLabelEXT pfn_vkQueueEndDebugUtilsLabelEXT = {}; + PFN_vkCmdBeginDebugUtilsLabelEXT pfn_vkCmdBeginDebugUtilsLabelEXT = {}; + PFN_vkCmdEndDebugUtilsLabelEXT pfn_vkCmdEndDebugUtilsLabelEXT = {}; + PFN_vkCmdInsertDebugUtilsLabelEXT pfn_vkCmdInsertDebugUtilsLabelEXT = {}; + + std::vector<size_t> device_indices; + std::vector<bool> device_supports_membudget; + vk_device devices[GGML_VK_MAX_DEVICES]; +}; + +typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + +static constexpr uint32_t kSpvOpCooperativeMatrixLoadTensorNV = 5367; + +static constexpr uint32_t kSpvCapabilityCooperativeMatrixDecodeVectorNV = 5447; + +static constexpr uint32_t kSpvTensorAddressingDecodeVectorFuncBit = 0x4; + +struct vk_fa_tuning_params { + FaCodePath path; + uint32_t workgroup_size; + uint32_t subgroup_size; + uint32_t block_rows; + uint32_t block_cols; + uint32_t d_split; + uint32_t row_split; + bool shmem_staging; + bool disable_subgroups; + uint32_t limit_occupancy_shmem; + + void print() const { + std::cerr << "path=" << path << " workgroup_size=" << workgroup_size << " subgroup_size=" << subgroup_size << + " block_rows=" << block_rows << " block_cols=" << block_cols << " d_split=" << d_split << + " row_split=" << row_split << " shmem_staging=" << shmem_staging << " disable_subgroups=" << disable_subgroups << + " limit_occupancy_shmem=" << limit_occupancy_shmem << std::endl; + } +}; + +struct GpuPipelineConfig { + // GPU architecture identifier. + // Example: vk_device_architecture::AMD_GCN + vk_device_architecture arch; + + // Mapping of pipeline names to their specific subgroup sizes. + // Example: {"soft_max_f32", 64} + std::unordered_map<std::string, uint32_t> pipelines; + + // Default subgroup size for this GPU. + // Defaults to 0 if not explicitly provided. + uint32_t default_subgroup_size = 0; +}; + +static constexpr uint32_t RDNA_DEFAULT_SUBGROUP_SIZE = 32; + +struct CompileTask { + vk_pipeline pipeline; + size_t spv_size; + const void * spv_data; + std::string entrypoint; + uint32_t parameter_count; + std::array<uint32_t, 3> wg_denoms; + std::vector<uint32_t> specialization_constants; + bool disable_robustness; + bool require_full_subgroups; + uint32_t required_subgroup_size; +}; + +struct ggml_vk_debug_label { + // at most one of these is set, depending on the scope the label was opened in + vk_context_struct * subctx {}; + vk_queue_handle * qhandle {}; + + // one region per dispatch, e.g. "matmul_q4_k_f32_f16acc_aligned_m (192,8,1)". + // RGP cannot recover the pipeline name on its own, it only has the hash + ggml_vk_debug_label(vk_context & ctx, const std::string & pipeline_name, uint32_t wg0, uint32_t wg1, uint32_t wg2); + + + // one region per graph node + // fused nodes are joined with '+', e.g. "RMS_NORM+MUL+ROPE Qcur-19" + ggml_vk_debug_label(vk_context & ctx, const ggml_cgraph * cgraph, int node_idx, int n_fused); + + + // one region per graph evaluation, opened on the queue instead of a command buffer + // so it spans every submit the evaluation makes + ggml_vk_debug_label(vk_queue_handle * handle, const char * name); + + + // call before the command buffer can end, the destructor covers the rest + void close(); + + + ~ggml_vk_debug_label() { + close(); + } + + ggml_vk_debug_label(const ggml_vk_debug_label &) = delete; + ggml_vk_debug_label & operator=(const ggml_vk_debug_label &) = delete; + +private: + // the constructors check this too, so the name is not built when markers are off + void begin(vk_context & ctx, const std::string & name); + +}; + +#define UNUSED GGML_UNUSED + +struct ggml_backend_vk_device_context { + size_t device; + std::string name; + std::string description; + bool is_integrated_gpu; + std::string pci_bus_id; + int op_offload_min_batch_size; +}; + diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 2d561d4cbf70..04538a4900e5 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1,456 +1,13 @@ -#include "ggml-vulkan.h" -#include <vulkan/vulkan_core.h> -#include <chrono> -#if defined(GGML_VULKAN_RUN_TESTS) || defined(GGML_VULKAN_CHECK_RESULTS) -#include "ggml-cpu.h" -#endif - -// See https://github.com/KhronosGroup/Vulkan-Hpp?tab=readme-ov-file#extensions--per-device-function-pointers- -#define VULKAN_HPP_DISPATCH_LOADER_DYNAMIC 1 -// We use VULKAN_HPP_DEFAULT_DISPATCHER, but not VULKAN_HPP_DEFAULT_DISPATCH_LOADER_DYNAMIC_STORAGE -// to avoid conflicts with applications or other libraries who might use it. -#if VK_HEADER_VERSION >= 301 -namespace vk::detail { class DispatchLoaderDynamic; } -using vk::detail::DispatchLoaderDynamic; -#else -namespace vk { class DispatchLoaderDynamic; } -using vk::DispatchLoaderDynamic; -#endif -DispatchLoaderDynamic & ggml_vk_default_dispatcher(); -#define VULKAN_HPP_DEFAULT_DISPATCHER ggml_vk_default_dispatcher() - -#include <vulkan/vulkan.hpp> - -// Fallback definitions for VK_NV_cooperative_matrix_decode_vector in case the -// installed Vulkan headers predate the extension. -#ifndef VK_NV_cooperative_matrix_decode_vector -#define VK_NV_cooperative_matrix_decode_vector 1 -#define VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME "VK_NV_cooperative_matrix_decode_vector" -#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV ((VkStructureType)1000689000) -typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV { - VkStructureType sType; - void* pNext; - VkBool32 cooperativeMatrixDecodeVector; -} VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV; -#endif - -// SPIR-V Headers: different SDK installations expose different include paths. -// LunarG Vulkan SDK on Windows typically provides <spirv-headers/spirv.hpp>. -// Linux packages, MSYS2 and MinGW often use the Khronos layout <spirv/unified1/spirv.hpp>. -#if __has_include(<spirv/unified1/spirv.hpp>) -# include <spirv/unified1/spirv.hpp> -#elif __has_include(<spirv-headers/spirv.hpp>) -# include <spirv-headers/spirv.hpp> -#elif __has_include(<spirv.hpp>) -# include <spirv.hpp> -#else - // Fallback to let the compiler throw a standard "file not found" error -# include <spirv/unified1/spirv.hpp> -#endif - -#include <algorithm> -#include <atomic> -#include <cmath> -#include <iomanip> -#include <iostream> -#include <tuple> -#include <vector> -#include <deque> -#include <sstream> -#include <utility> -#include <memory> -#include <limits> -#include <map> -#include <set> -#include <unordered_map> -#include <shared_mutex> -#include <mutex> -#include <future> -#include <condition_variable> -#include <thread> - -#if defined(_MSC_VER) -# define NOMINMAX 1 -# include <windows.h> -# define YIELD() YieldProcessor() -#elif defined(__clang__) || defined(__GNUC__) -# if defined(__x86_64__) ||defined(__i386__) -# include <immintrin.h> -# define YIELD() _mm_pause() -# elif defined(__arm__) || defined(__aarch64__) -# if defined(__clang__) -# include <arm_acle.h> -# define YIELD() __yield() -# else -# define YIELD() asm volatile("yield") -# endif -# endif -#endif - -#if !defined(YIELD) -#define YIELD() -#endif - -#include "ggml-impl.h" -#include "ggml-backend-impl.h" - -#include "ggml-vulkan-shaders.hpp" - -// remove this once it's more widely available in the SDK -#if !defined(VK_KHR_shader_bfloat16) - -#define VK_KHR_shader_bfloat16 1 -#define VK_KHR_SHADER_BFLOAT16_SPEC_VERSION 1 -#define VK_KHR_SHADER_BFLOAT16_EXTENSION_NAME "VK_KHR_shader_bfloat16" -#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR ((VkStructureType)1000141000) -#define VK_COMPONENT_TYPE_BFLOAT16_KHR ((VkComponentTypeKHR)1000141000) - -typedef struct VkPhysicalDeviceShaderBfloat16FeaturesKHR { - VkStructureType sType; - void* pNext; - VkBool32 shaderBFloat16Type; - VkBool32 shaderBFloat16DotProduct; - VkBool32 shaderBFloat16CooperativeMatrix; -} VkPhysicalDeviceShaderBfloat16FeaturesKHR; -#endif - -#if !defined(VK_VALVE_shader_mixed_float_dot_product) -#define VK_VALVE_shader_mixed_float_dot_product 1 -#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_SPEC_VERSION 1 -#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_EXTENSION_NAME "VK_VALVE_shader_mixed_float_dot_product" -#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE ((VkStructureType)1000673000) -typedef struct VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE { - VkStructureType sType; - void* pNext; - VkBool32 shaderMixedFloatDotProductFloat16AccFloat32; - VkBool32 shaderMixedFloatDotProductFloat16AccFloat16; - VkBool32 shaderMixedFloatDotProductBFloat16Acc; - VkBool32 shaderMixedFloatDotProductFloat8AccFloat32; -} VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE; -#endif - -#if !defined(VK_EXT_shader_ocp_microscaling_types) -#define VK_EXT_shader_ocp_microscaling_types 1 -#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_SPEC_VERSION 1 -#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME "VK_EXT_shader_ocp_microscaling_types" -#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OCP_MICROSCALING_TYPES_FEATURES_EXT ((VkStructureType)1000672000) -typedef struct VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT { - VkStructureType sType; - void* pNext; - VkBool32 shaderFloat4; - VkBool32 shaderFloat6; - VkBool32 shaderFloat8UnsignedE8M0; - VkBool32 shaderMXInt8; -} VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT; -#endif - -#if !defined(VK_EXT_shader_float8) -#define VK_EXT_shader_float8 1 -#define VK_EXT_SHADER_FLOAT8_SPEC_VERSION 1 -#define VK_EXT_SHADER_FLOAT8_EXTENSION_NAME "VK_EXT_shader_float8" -#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT ((VkStructureType)1000567000) -typedef struct VkPhysicalDeviceShaderFloat8FeaturesEXT { - VkStructureType sType; - void* pNext; - VkBool32 shaderFloat8; - VkBool32 shaderFloat8CooperativeMatrix; -} VkPhysicalDeviceShaderFloat8FeaturesEXT; -#endif - -#ifndef VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME -#define VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME "VK_KHR_internally_synchronized_queues" -#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INTERNALLY_SYNCHRONIZED_QUEUES_FEATURES_KHR ((VkStructureType)1000504000) -#define VK_DEVICE_QUEUE_CREATE_INTERNALLY_SYNCHRONIZED_BIT_KHR ((VkDeviceQueueCreateFlagBits)0x00000004) - -// Compile-time constant guaranteed; no runtime initialization overhead -static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR = - static_cast<vk::DeviceQueueCreateFlagBits>(0x00000004); - -typedef struct VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR { - VkStructureType sType; - void* pNext; - VkBool32 internallySynchronizedQueues; -} VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR; -#else -static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR = vk::DeviceQueueCreateFlagBits::eInternallySynchronizedKHR; -#endif - -#define ROUNDUP_POW2(M, N) (((M) + (N) - 1) & ~((N) - 1)) -#define CEIL_DIV(M, N) (((M) / (N)) + (((M) % (N)) != 0)) -static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; } - -#define VK_VENDOR_ID_AMD 0x1002 -#define VK_VENDOR_ID_APPLE 0x106b -#define VK_VENDOR_ID_INTEL 0x8086 -#define VK_VENDOR_ID_NVIDIA 0x10de -#define VK_VENDOR_ID_QUALCOMM 0x5143 - -#define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256 - -#define VK_CHECK(err, msg, dev) \ - do { \ - vk::Result err_; \ - try { \ - err_ = (err); \ - } catch (vk::DeviceLostError &) { \ - ggml_vk_print_device_lost_info(dev); \ - GGML_LOG_ERROR("ggml_vulkan: %s at %s:%d\n", \ - #err, __FILE__, __LINE__); \ - throw; \ - } \ - if (err_ != vk::Result::eSuccess) { \ - GGML_LOG_ERROR("ggml_vulkan: %s error %s at %s:%d\n", \ - #err, to_string(err_).c_str(), __FILE__, __LINE__); \ - throw vk::SystemError(vk::make_error_code(err_), \ - "ggml_vulkan: " msg); \ - } \ - } while (0) - -#ifdef GGML_VULKAN_DEBUG -#define VK_LOG_DEBUG(msg) std::cerr << msg << std::endl -#else -#define VK_LOG_DEBUG(msg) ((void) 0) -#endif // GGML_VULKAN_DEBUG - -struct ggml_backend_vk_context; - -#define MAX_PARAMETER_COUNT 12 -// Max number of adds that can be fused without exceeding MAX_PARAMETER_COUNT. -#define MAX_FUSED_ADDS (MAX_PARAMETER_COUNT - 3) - -typedef std::shared_ptr<struct vk_pipeline_struct> vk_pipeline; - -struct vk_pipeline_struct { - std::string name; - vk::ShaderModule shader_module; - vk::PipelineLayout layout; - vk::Pipeline pipeline; - uint32_t push_constant_size; - uint32_t parameter_count; - std::array<uint32_t, 3> wg_denoms; - uint32_t align; - // true if fields have been set by ggml_vk_create_pipeline - bool initialized {}; - // true while a compile is in flight, used to dedupe concurrent claims. - // Protected by device->compile_mutex. - bool compile_pending {}; - // set to true when the shader has been compiled - std::atomic<bool> compiled {}; - // number of registers used, extracted from pipeline executable properties - uint32_t register_count {}; - -#if defined(VK_EXT_shader_64bit_indexing) - bool is_64b_indexing {}; -#endif - // linked list of pipelines for multiple compilation variants. - // currently only used to compile a 64-bit indexing variant. - vk_pipeline next; -}; - -typedef std::weak_ptr<vk_pipeline_struct> vk_pipeline_ref; - -static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline); - -struct vk_matmul_pipeline_struct { - vk_pipeline l, m, s; - vk_pipeline a_l, a_m, a_s; - // Returns true when all unaligned pipelines are null. - // We only check for unaligned variants since one of the unaligned pipelines must exist - // while aligned pipelines are optional - bool is_empty() const { - return l == nullptr && m == nullptr && s == nullptr; - } -}; -typedef std::shared_ptr<vk_matmul_pipeline_struct> vk_matmul_pipeline; - -struct vk_matmul_pipeline2 { - vk_matmul_pipeline2() { - f16acc = std::make_shared<vk_matmul_pipeline_struct>(); - f32acc = std::make_shared<vk_matmul_pipeline_struct>(); - } - vk_matmul_pipeline f32acc; - vk_matmul_pipeline f16acc; -}; - -struct vk_device_struct; -typedef std::shared_ptr<vk_device_struct> vk_device; -typedef std::weak_ptr<vk_device_struct> vk_device_ref; - -struct vk_buffer_struct; -typedef std::shared_ptr<vk_buffer_struct> vk_buffer; -typedef std::weak_ptr<vk_buffer_struct> vk_buffer_ref; - -struct ggml_backend_vk_buffer_type_context { - std::string name; - vk_device device; -}; +#include "ggml-vulkan-common.h" -struct vk_queue; - -struct vk_command_buffer { - vk::CommandBuffer buf; - uint64_t use_counter = 0; - bool in_use = false; -}; - -// Stores command pool/buffers. There's an instance of this -// for each (context,queue) pair and for each (device,queue) pair. -struct vk_command_pool { - void init(vk_device& device, vk_queue *q_); - void destroy(vk::Device& device); - - vk::CommandPool pool; - // Using deque so the pointers to command buffers - // remain valid even if we add more - std::deque<vk_command_buffer> cmd_buffers; - - vk_queue *q; - - size_t buffers_in_use() const { - return std::count_if(cmd_buffers.begin(), cmd_buffers.end(), - [](const auto& cb) { return cb.in_use; }); - } -}; - -static void ggml_vk_print_device_fault_info(const vk_device& device); -static void ggml_vk_print_device_lost_info(const vk_device& device); - -// Prevent simultaneous submissions to the same queue. -struct vk_queue_handle { - vk::Queue queue; - vk_device_ref device; - // *** QUEUE QUIESCENCE EPOCHS (bookkeeping only -- inert on its own). *** - // - // submit_epoch counts every vkQueueSubmit ever issued through this handle; - // it is bumped BEFORE the submit so an observer never sees the two counters - // equal while a batch is in flight. waited_epoch is advanced (monotonically) - // to the submit_epoch value captured just before a fence wait that covers - // everything submitted to this queue so far. submit_epoch == waited_epoch - // therefore means "this queue has no work outstanding that anyone still has - // to wait for", which is what lets WP_VK_HOST_READ_FASTPATH read - // host-coherent memory with a plain memcpy instead of a barrier submit. - // - // Missing a publish point only makes the predicate pessimistic (the fast - // path stops engaging); missing a BUMP would make it unsound, which is why - // the bump lives in the submit overrides -- the single choke point for - // every vkQueueSubmit in this file -- and not at the call sites. - std::atomic<uint64_t> submit_epoch {0}; - std::atomic<uint64_t> waited_epoch {0}; - virtual void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) = 0; - virtual void lock() {} // no-op by default (internally synchronized case) - virtual void unlock() {} - virtual ~vk_queue_handle() = default; -}; - -// True when nothing submitted to this queue is still awaiting a wait. -static inline bool ggml_vk_queue_quiesced(const vk_queue_handle & h) { - return h.waited_epoch.load(std::memory_order_acquire) == - h.submit_epoch.load(std::memory_order_acquire); -} +static void ggml_vk_node_ts_harvest(ggml_backend_vk_context * ctx); +static void ggml_vk_flush_cpy_regions(vk_context& ctx); -// Capture the epoch to publish once a covering fence wait returns. -static inline uint64_t ggml_vk_queue_epoch(const vk_queue_handle & h) { - return h.submit_epoch.load(std::memory_order_acquire); +namespace { +inline std::ostream & operator<<(std::ostream & os, vk::Buffer buffer) { + return os << static_cast<VkBuffer>(buffer); } - -// Publish (monotonically) that everything up to `epoch` has been waited on. -static inline void ggml_vk_queue_mark_waited(vk_queue_handle & h, uint64_t epoch) { - uint64_t cur = h.waited_epoch.load(std::memory_order_relaxed); - while (cur < epoch && - !h.waited_epoch.compare_exchange_weak(cur, epoch, - std::memory_order_release, - std::memory_order_relaxed)) { - } } - -struct vk_queue_handle_synchronized : vk_queue_handle { - std::mutex mutex; - void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override { - std::lock_guard<std::mutex> guard(mutex); - submit_epoch.fetch_add(1, std::memory_order_acq_rel); - try { - queue.submit(submits, fence); - } catch (vk::DeviceLostError &) { - if (auto dev = device.lock()) { - ggml_vk_print_device_lost_info(dev); - } - throw; - } - } - void lock() override { mutex.lock(); } - void unlock() override { mutex.unlock(); } -}; - -struct vk_queue_handle_unsynchronized : vk_queue_handle { - void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override { - // Driver guarantees internal synchronization via VK_KHR_internally_synchronized_queues - submit_epoch.fetch_add(1, std::memory_order_acq_rel); - try { - queue.submit(submits, fence); - } catch (vk::DeviceLostError &) { - if (auto dev = device.lock()) { - ggml_vk_print_device_lost_info(dev); - } - throw; - } - } - // lock()/unlock() inherited no-ops -}; - -struct vk_queue { - uint32_t queue_family_index; - std::shared_ptr<vk_queue_handle> handle; - - vk_command_pool cmd_pool; - - vk::PipelineStageFlags stage_flags; - - bool transfer_only; -}; - -static const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft); -static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size); -static size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft); -static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft); -static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor); -static ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = { - /* .get_name = */ ggml_backend_vk_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_vk_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_vk_buffer_type_get_max_size, - /* .get_alloc_size = */ ggml_backend_vk_buffer_type_get_alloc_size, - /* .is_host = */ NULL, -}; - -class vk_memory_logger; -class vk_perf_logger; -static void ggml_vk_destroy_buffer(vk_buffer& buf); -static void ggml_vk_synchronize(ggml_backend_vk_context * ctx); -static void ggml_vk_node_ts_harvest(ggml_backend_vk_context * ctx); -static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size); -static bool ggml_vk_is_empty(ggml_tensor * node); - -// Upper bound on the batch width the mat-vec (GEMV) path can serve, and hence -// the size of the per-NUM_COLS pipeline arrays. The *effective* cap is per -// device (vk_device::mul_mat_vec_max_cols): only architectures that have been -// measured to stay below the mul_mm path at these widths go above 8, so nobody -// else pays the pipeline-compile cost for the extra instantiations. -static constexpr uint32_t mul_mat_vec_max_cols = 16; -static constexpr uint32_t mul_mat_vec_max_cols_default = 8; -static constexpr uint32_t p021_max_gqa_ratio = 8; - -enum vk_device_architecture { - OTHER, - AMD_GCN, - AMD_RDNA1, - AMD_RDNA2, - AMD_RDNA3, - INTEL_XE1, - INTEL_XE2, - NVIDIA_PRE_TURING, - NVIDIA_TURING, -}; - static vk_device_architecture get_device_architecture(const vk::PhysicalDevice& device) { vk::PhysicalDeviceProperties props = device.getProperties(); @@ -571,726 +128,10 @@ static vk_device_architecture get_device_architecture(const vk::PhysicalDevice& return vk_device_architecture::OTHER; } -enum vk_conv_shapes { - CONV_SHAPE_128x128, - CONV_SHAPE_64x32, - CONV_SHAPE_32x256, - CONV_SHAPE_64x128, - CONV_SHAPE_COUNT, -}; - -struct vk_conv_block_size { - uint32_t K; - uint32_t NPQ; - uint32_t CRS; -}; - -vk_conv_block_size vk_conv_block_sizes[CONV_SHAPE_COUNT] = { - // K NPQ CRS - { 128, 128, 16 }, // CONV_SHAPE_128x128 - { 64, 32, 32 }, // CONV_SHAPE_64x32 - { 32, 256, 16 }, // CONV_SHAPE_32x256 - { 64, 128, 16 }, // CONV_SHAPE_64x128 -}; - -enum dmmv_wg_sizes { - DMMV_WG_SIZE_SUBGROUP, - DMMV_WG_SIZE_LARGE, - DMMV_WG_SIZE_COUNT, -}; - -enum FaCodePath { - FA_SCALAR, - FA_COOPMAT1, - FA_COOPMAT2, -}; - -struct vk_fa_pipeline_state { - uint32_t HSK, HSV; - uint32_t Br, Bc; - uint32_t D_split, row_split; - bool shmem_staging; - FaCodePath path; - uint32_t workgroup_size, subgroup_size; - bool aligned; - bool f32acc; - uint32_t flags; - uint32_t limit_occupancy_shmem; - ggml_type k_type; - ggml_type v_type; - - bool operator<(const vk_fa_pipeline_state &b) const { - return std::tie(HSK, HSV, Br, Bc, D_split, row_split, shmem_staging, path, workgroup_size, subgroup_size, aligned, f32acc, flags, limit_occupancy_shmem, k_type, v_type) < - std::tie(b.HSK, b.HSV, b.Br, b.Bc, b.D_split, b.row_split, b.shmem_staging, b.path, b.workgroup_size, b.subgroup_size, b.aligned, b.f32acc, b.flags, b.limit_occupancy_shmem, b.k_type, b.v_type); - } -}; - -struct vk_conv2d_pipeline_state { - vk_conv2d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t p0, uint32_t p1, uint32_t d0, uint32_t d1, uint32_t KW, uint32_t KH, uint32_t aligned) - : s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), KW(KW), KH(KH), aligned(aligned) {} - - uint32_t s0, s1, p0, p1, d0, d1, KW, KH; - // when set, shader can skip K/CRS/NPQ bounds checks and address clamps - uint32_t aligned; - - bool operator<(const vk_conv2d_pipeline_state &b) const { - return std::tie(s0, s1, p0, p1, d0, d1, KW, KH, aligned) < - std::tie(b.s0, b.s1, b.p0, b.p1, b.d0, b.d1, b.KW, b.KH, b.aligned); - } -}; - -struct vk_conv3d_pipeline_state { - vk_conv3d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t s2, uint32_t p0, uint32_t p1, uint32_t p2, - uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned) - : s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned) {} - - uint32_t s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD; - uint32_t aligned; - - bool operator<(const vk_conv3d_pipeline_state &b) const { - return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned) < - std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned); - } -}; - -struct vk_solve_tri_pipeline_state { - vk_solve_tri_pipeline_state(uint32_t N, uint32_t K) - : N(N), K(K) {} - - uint32_t N, K; - - bool operator<(const vk_solve_tri_pipeline_state &b) const { - return std::tie(N, K) < - std::tie(b.N, b.K); - } -}; - -enum shader_reduction_mode { - SHADER_REDUCTION_MODE_SHMEM, - SHADER_REDUCTION_MODE_HYBRID, - SHADER_REDUCTION_MODE_SUBGROUP, - SHADER_REDUCTION_MODE_COUNT, -}; - -// argsort pipelines for up to 1<<10 invocations per workgroup -static constexpr uint32_t num_argsort_pipelines = 11; -static constexpr uint32_t num_topk_moe_pipelines = 10; -static constexpr uint32_t num_topk_pipelines = 11; - -static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax_norm{ GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, - GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE, - GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, - GGML_OP_RESHAPE }; - -static constexpr std::initializer_list<ggml_op> topk_moe_sigmoid_norm_bias{ GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, - GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, - GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, - GGML_OP_DIV, GGML_OP_RESHAPE }; - -static constexpr std::initializer_list<ggml_op> topk_moe_sqrt_softplus_norm_bias{ GGML_OP_UNARY, GGML_OP_SQRT, - GGML_OP_RESHAPE, GGML_OP_ADD, - GGML_OP_ARGSORT, GGML_OP_VIEW, - GGML_OP_GET_ROWS, GGML_OP_RESHAPE, - GGML_OP_SUM_ROWS, GGML_OP_CLAMP, - GGML_OP_DIV, GGML_OP_RESHAPE }; - -static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, - GGML_OP_VIEW, GGML_OP_GET_ROWS }; - -static constexpr std::initializer_list<ggml_op> topk_moe_late_softmax { GGML_OP_ARGSORT, GGML_OP_VIEW, - GGML_OP_GET_ROWS, GGML_OP_RESHAPE, - GGML_OP_SOFT_MAX, GGML_OP_RESHAPE }; - -// Snake activation: y = x + sin(a*x)^2 * inv_b. Used by the optimize_graph reorder -// pass so it keeps the chain contiguous and by the dispatcher to detect the fusion. -static constexpr std::initializer_list<ggml_op> snake_pattern { GGML_OP_MUL, GGML_OP_SIN, - GGML_OP_SQR, GGML_OP_MUL, - GGML_OP_ADD }; - -// qwen4 QSA indexer: gather per-block scores to cells + add f16 mask (cast+reshape) + top-k, -// fused into one radix-select. The cast/reshape are elided; the raw f16 mask is read in-shader. -static constexpr std::initializer_list<ggml_op> topk_qsa_pattern { GGML_OP_GET_ROWS, GGML_OP_PERMUTE, - GGML_OP_CONT, GGML_OP_CPY, - GGML_OP_RESHAPE, GGML_OP_ADD, - GGML_OP_TOP_K }; -static constexpr std::initializer_list<std::array<int, 3>> topk_qsa_edges { - { 1, 0, 0 }, // permute->src[0] == get_rows - { 2, 0, 1 }, // cont->src[0] == permute - { 4, 0, 3 }, // reshape->src[0] == cpy (mask cast) - { 5, 0, 2 }, // add->src[0] == cont - { 5, 1, 4 }, // add->src[1] == reshape - { 6, 0, 5 }, // top_k->src[0] == add -}; - -//node #978 ( SOFT_MAX): ffn_moe_probs-15 ( 0K) [Vulka ] use=2: ffn_moe_logits-15 ( 0K) [Vulka ] -//node #979 ( RESHAPE): ffn_moe_probs-15 (re ( 0K) [Vulka ] use=1: ffn_moe_probs-15 ( 0K) [Vulka ] -//node #980 ( ARGSORT): ffn_moe_argsort-15 ( 0K) [Vulka ] use=1: ffn_moe_probs-15 ( 0K) [Vulka ] -//node #981 ( VIEW): ffn_moe_topk-15 ( 0K) [Vulka ] use=4: ffn_moe_argsort-15 ( 0K) [Vulka ] -//node #982 ( GET_ROWS): ffn_moe_weights-15 ( 0K) [Vulka ] use=1: ffn_moe_probs-15 (re ( 0K) [Vulka ] ffn_moe_topk-15 ( 0K) [Vulka ] -//node #983 ( RESHAPE): ffn_moe_weights-15 ( ( 0K) [Vulka ] use=2: ffn_moe_weights-15 ( 0K) [Vulka ] -//node #984 ( SUM_ROWS): ffn_moe_weights_sum- ( 0K) [Vulka ] use=1: ffn_moe_weights-15 ( ( 0K) [Vulka ] -//node #985 ( CLAMP): ffn_moe_weights_sum_ ( 0K) [Vulka ] use=1: ffn_moe_weights_sum- ( 0K) [Vulka ] -//node #986 ( DIV): ffn_moe_weights_norm ( 0K) [Vulka ] use=1: ffn_moe_weights-15 ( ( 0K) [Vulka ] ffn_moe_weights_sum_ ( 0K) [Vulka ] -//node #987 ( RESHAPE): ffn_moe_weights_norm ( 0K) [Vulka ] use=1: ffn_moe_weights_norm ( 0K) [Vulka ] -static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_norm_edges { - { 1, 0, 0 }, // reshape->src[0] == softmax - { 2, 0, 0 }, // argsort->src[0] == softmax - { 3, 0, 2 }, // view->src[0] == argsort - { 4, 0, 1 }, // get_rows->src[0] == reshape - { 4, 1, 3 }, // get_rows->src[1] == view - { 5, 0, 4 }, // reshape->src[0] == get_rows - { 6, 0, 5 }, // sum_rows->src[0] == reshape - { 7, 0, 6 }, // clamp->src[0] == sum_rows - { 8, 0, 5 }, // div->src[0] == reshape - { 8, 1, 7 }, // div->src[1] == clamp - { 9, 0, 8 }, // reshape->src[0] == div -}; - -//node #436 ( UNARY): ffn_moe_probs-10 ( 256K) [Vulka ] use=2: ffn_moe_logits-10 ( 256K) [Vulka ] -//node #437 ( RESHAPE): ffn_moe_probs-10 (re ( 256K) [Vulka ] use=1: ffn_moe_probs-10 ( 256K) [Vulka ] -//node #438 ( ADD): ffn_moe_probs_biased ( 256K) [Vulka ] use=1: ffn_moe_probs-10 ( 256K) [Vulka ] blk.10.exp_probs_b.b ( 0K) [Vulka ] -//node #439 ( ARGSORT): ffn_moe_argsort-10 ( 256K) [Vulka ] use=1: ffn_moe_probs_biased ( 256K) [Vulka ] -//node #440 ( VIEW): ffn_moe_topk-10 ( 255K) [Vulka ] use=3: ffn_moe_argsort-10 ( 256K) [Vulka ] -//node #441 ( GET_ROWS): ffn_moe_weights-10 ( 12K) [Vulka ] use=1: ffn_moe_probs-10 (re ( 256K) [Vulka ] ffn_moe_topk-10 ( 255K) [Vulka ] -//node #442 ( RESHAPE): ffn_moe_weights-10 ( ( 12K) [Vulka ] use=2: ffn_moe_weights-10 ( 12K) [Vulka ] -//node #443 ( SUM_ROWS): ffn_moe_weights_sum- ( 2K) [Vulka ] use=1: ffn_moe_weights-10 ( ( 12K) [Vulka ] -//node #444 ( CLAMP): ffn_moe_weights_sum_ ( 2K) [Vulka ] use=1: ffn_moe_weights_sum- ( 2K) [Vulka ] -//node #445 ( DIV): ffn_moe_weights_norm ( 12K) [Vulka ] use=1: ffn_moe_weights-10 ( ( 12K) [Vulka ] ffn_moe_weights_sum_ ( 2K) [Vulka ] -//node #446 ( RESHAPE): ffn_moe_weights_norm ( 12K) [Vulka ] use=1: ffn_moe_weights_norm ( 12K) [Vulka ] -static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sigmoid_norm_bias_edges { - { 1, 0, 0 }, // reshape->src[0] == sigmoid - { 2, 0, 0 }, // add->src[0] == sigmoid - { 3, 0, 2 }, // argsort->src[0] == add - { 4, 0, 3 }, // view->src[0] == argsort - { 5, 0, 1 }, // get_rows->src[0] == reshape - { 5, 1, 4 }, // get_rows->src[1] == view - { 6, 0, 5 }, // reshape->src[0] == get_rows - { 7, 0, 6 }, // sum_rows->src[0] == reshape - { 8, 0, 7 }, // clamp->src[0] == sum_rows - { 9, 0, 6 }, // div->src[0] == reshape - { 9, 1, 8 }, // div->src[1] == clamp - {10, 0, 9 }, // reshape->src[0] == div -}; - -static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sqrt_softplus_norm_bias_edges { - { 1, 0, 0 }, // sqrt->src[0] == softplus - { 2, 0, 1 }, // reshape->src[0] == sqrt - { 3, 0, 1 }, // add->src[0] == sqrt - { 4, 0, 3 }, // argsort->src[0] == add - { 5, 0, 4 }, // view->src[0] == argsort - { 6, 0, 2 }, // get_rows->src[0] == reshape - { 6, 1, 5 }, // get_rows->src[1] == view - { 7, 0, 6 }, // reshape->src[0] == get_rows - { 8, 0, 7 }, // sum_rows->src[0] == reshape - { 9, 0, 8 }, // clamp->src[0] == sum_rows - {10, 0, 7 }, // div->src[0] == reshape - {10, 1, 9 }, // div->src[1] == clamp - {11, 0,10 }, // reshape->src[0] == div -}; - -// same as early_softmax_norm but ending after the get_rows -static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_edges { - { 1, 0, 0 }, // reshape->src[0] == softmax - { 2, 0, 0 }, // argsort->src[0] == softmax - { 3, 0, 2 }, // view->src[0] == argsort - { 4, 0, 1 }, // get_rows->src[0] == reshape - { 4, 1, 3 }, // get_rows->src[1] == view -}; - -//node #652 ( ARGSORT): ffn_moe_argsort-11 ( 0K) [Vulka ] use=1: ffn_moe_probs-11 ( 0K) [Vulka ] -//node #653 ( VIEW): ffn_moe_topk-11 ( 0K) [Vulka ] use=7: ffn_moe_argsort-11 ( 0K) [Vulka ] -//node #654 ( GET_ROWS): ffn_moe_weights-11 ( 0K) [Vulka ] use=1: ffn_moe_probs-11 (re ( 0K) [Vulka ] ffn_moe_topk-11 ( 0K) [Vulka ] -//node #655 ( RESHAPE): ffn_moe_weights-11 ( ( 0K) [Vulka ] use=1: ffn_moe_weights-11 ( 0K) [Vulka ] -//node #656 ( SOFT_MAX): node_656 ( 0K) [Vulka ] use=1: ffn_moe_weights-11 ( ( 0K) [Vulka ] -//node #657 ( RESHAPE): ffn_moe_weights_soft ( 0K) [Vulka ] use=1: node_656 ( 0K) [Vulka ] -static constexpr std::initializer_list<std::array<int, 3>> topk_moe_late_softmax_edges { - { 1, 0, 0 }, // view->src[0] == argsort - { 2, 1, 1 }, // get_rows->src[1] == view - { 3, 0, 2 }, // reshape->src[0] == get_rows - { 4, 0, 3 }, // soft_max->src[0] == reshape - { 5, 0, 4 }, // reshape->src[0] == soft_max -}; - -enum topk_moe_mode { - TOPK_MOE_EARLY_SOFTMAX, - TOPK_MOE_EARLY_SOFTMAX_NORM, - TOPK_MOE_LATE_SOFTMAX, - TOPK_MOE_SIGMOID_NORM_BIAS, - TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS, - TOPK_MOE_COUNT, -}; - -static constexpr std::initializer_list<std::array<int, 3>> rope_view_set_rows_edges { - { 1, 0, 0 }, // view->src[0] == rope - { 2, 0, 1 }, // set_rows->src[0] == view -}; - -static constexpr std::initializer_list<std::array<int, 3>> rms_norm_mul_rope_view_set_rows_edges { - { 1, 0, 0 }, // mul->src[0] == rms - { 2, 0, 1 }, // rope->src[0] == mul - { 3, 0, 2 }, // view->src[0] == rope - { 4, 0, 3 }, // set_rows->src[0] == view -}; - -static constexpr std::array<ggml_type, 9> lightning_indexer_k_types = { - GGML_TYPE_F32, - GGML_TYPE_F16, - GGML_TYPE_BF16, - GGML_TYPE_Q8_0, - GGML_TYPE_Q5_1, - GGML_TYPE_Q5_0, - GGML_TYPE_Q4_1, - GGML_TYPE_Q4_0, - GGML_TYPE_IQ4_NL, -}; - -static bool ggml_vk_lightning_indexer_k_type_supported(ggml_type type) { +bool ggml_vk_lightning_indexer_k_type_supported(ggml_type type) { return std::find(lightning_indexer_k_types.begin(), lightning_indexer_k_types.end(), type) != lightning_indexer_k_types.end(); } - -struct vk_device_struct { - std::recursive_mutex mutex; - mutable std::shared_mutex pinned_memory_mutex; - - // Guards compile_pending, all_pipelines, and the dynamic pipeline maps - // (flash_attn, fa_mask_opt, solve_tri, conv2d, etc). The actual compile - // runs with no lock held, so different pipelines can compile in parallel. - // Lock order is device->mutex -> compile_mutex, never the reverse. - std::mutex compile_mutex; - std::condition_variable compile_cv; - - vk::PhysicalDevice physical_device; - vk::PhysicalDeviceProperties properties; - std::string name; - uint64_t max_memory_allocation_size; - uint64_t max_buffer_size; - uint64_t suballocation_block_size; - uint64_t min_imported_host_pointer_alignment; - bool external_memory_host {}; - // VK_KHR_external_semaphore_fd: lets two vk::Devices in this process share - // one timeline semaphore payload, which the tensor-parallel AllReduce - // (ggml-vulkan-allreduce.hpp) uses for cross-device ordering. - bool external_semaphore_fd {}; - bool fp16; - bool bf16; - bool pipeline_robustness; - bool memory_priority; - vk::Device device; - uint32_t vendor_id; - vk::DriverId driver_id; - vk_device_architecture architecture; - std::unique_ptr<vk_queue> compute_queue; - std::unique_ptr<vk_queue> transfer_queue; - bool single_queue; - bool support_async; - bool async_use_transfer_queue; - bool has_internally_synchronized_queues = false; - uint32_t subgroup_size; - uint32_t subgroup_size_log2; - uint32_t shader_core_count; - // Effective GEMV batch-width cap for this device, <= mul_mat_vec_max_cols. - // Set in ggml_vk_load_shaders before any mat-vec pipeline is created. - uint32_t mul_mat_vec_max_cols_eff = mul_mat_vec_max_cols_default; - bool uma; - bool prefer_host_memory; - bool float_controls_rte_fp16; - bool float_controls_denorm_preserve_fp16; - bool subgroup_basic; - bool subgroup_arithmetic; - bool subgroup_shuffle; - bool subgroup_ballot; - bool subgroup_clustered; - bool subgroup_vote; - bool multi_add; - bool shader_int64; - bool buffer_device_address; - bool vulkan_memory_model; - - bool add_rms_fusion; - uint32_t partials_binding_alignment; - uint32_t max_nodes_per_submit; - - bool shader_64b_indexing; - - bool integer_dot_product; - // 0: default, 1: force mmvq, -1: disable mmvq - int32_t mmvq_mode; - - bool subgroup_size_control; - uint32_t subgroup_min_size; - uint32_t subgroup_max_size; - bool subgroup_require_full_support; - - // floor(log2(maxComputeWorkGroupInvocations)) - uint32_t max_workgroup_size_log2 {}; - - bool coopmat_support; - bool coopmat_acc_f32_support {}; - bool coopmat_acc_f16_support {}; - bool coopmat_bf16_support {}; - bool coopmat_support_16x16x16_f16acc {}; - bool coopmat_support_16x16x16_f32acc {}; - bool coopmat1_fa_support {}; - uint32_t coopmat_m; - uint32_t coopmat_n; - uint32_t coopmat_k; - - bool coopmat_int_support; - uint32_t coopmat_int_m; - uint32_t coopmat_int_n; - uint32_t coopmat_int_k; - - bool coopmat2; - bool coopmat2_bf16_support {}; - bool coopmat2_decode_vector; - - bool dot2_f16 {}; - bool ocp_fp4 {}; - - bool pipeline_executable_properties_support {}; - - bool device_fault {}; - PFN_vkGetDeviceFaultInfoEXT pfn_vkGetDeviceFaultInfoEXT {}; - - bool serialize_submissions {}; - - const ggml_cgraph * diag_cgraph {}; - int diag_prev_start = -1; - int diag_prev_end = -1; - - size_t idx; - - bool mul_mat_l[GGML_TYPE_COUNT]; - bool mul_mat_m[GGML_TYPE_COUNT]; - bool mul_mat_s[GGML_TYPE_COUNT]; - bool mul_mat_id_l[GGML_TYPE_COUNT]; - bool mul_mat_id_m[GGML_TYPE_COUNT]; - bool mul_mat_id_s[GGML_TYPE_COUNT]; - - // Separate flags for the q8_1 (integer dot) mmq path, whose shader uses - // a different shared-memory layout than the float matmul shaders. - bool mul_mat_l_int[GGML_TYPE_COUNT]; - bool mul_mat_m_int[GGML_TYPE_COUNT]; - bool mul_mat_s_int[GGML_TYPE_COUNT]; - bool mul_mat_id_l_int[GGML_TYPE_COUNT]; - bool mul_mat_id_m_int[GGML_TYPE_COUNT]; - bool mul_mat_id_s_int[GGML_TYPE_COUNT]; - - vk::DescriptorSetLayout dsl; - - vk_matmul_pipeline pipeline_matmul_f32 {}; - vk_matmul_pipeline pipeline_matmul_f32_f16 {}; - vk_matmul_pipeline pipeline_matmul_bf16 {}; - vk_matmul_pipeline2 pipeline_matmul_f16; - vk_matmul_pipeline2 pipeline_matmul_f16_f32; - - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat[GGML_TYPE_COUNT]; - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_COUNT]; - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_COUNT]; - - vk_matmul_pipeline pipeline_matmul_id_f32 {}; - vk_matmul_pipeline pipeline_matmul_id_bf16 {}; - vk_matmul_pipeline2 pipeline_matmul_id_f16; - vk_matmul_pipeline2 pipeline_matmul_id_f16_f32; - - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_id[GGML_TYPE_COUNT]; - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_COUNT]; - - vk_pipeline pipeline_matmul_split_k_reduce; - vk_pipeline pipeline_quantize_q8_1_x4; - - vk_pipeline pipeline_dequant[GGML_TYPE_COUNT]; - vk_pipeline pipeline_dequant_transpose[GGML_TYPE_COUNT]; // fused dequant+transpose for FA quant-KV - vk_pipeline pipeline_dequant_mul_mat_vec_f32_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols]; - vk_pipeline pipeline_dequant_mul_mat_vec_f16_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols]; - vk_pipeline pipeline_dequant_mul_mat_vec_id_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT]; - - vk_pipeline pipeline_dequant_mul_mat_vec_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols]; - vk_pipeline pipeline_dequant_mul_mat_vec_id_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT]; - - vk_pipeline pipeline_wp_fused_expert_q5_1; - vk_pipeline pipeline_wp_fused_expert_q8_0; - // Batched fast path (see ggml_backend_vk_wp_fused_expert): TWO shaders, - // one dispatch each, covering every selected expert of one request - // instead of one dispatch per expert. Deliberately two separate - // pipelines/shader modules (not one shader branching on a `phase` push - // constant, as an earlier version did) so phase B contains no - // barrier()/shared-memory staging that depended on every invocation of - // the workgroup -- including masked-out ones -- reaching it; that - // design hung the RX 480 (RADV/ACO) even after replacing an - // early-return-before-barrier with a clamp. Phase A never touches the - // down buffer, so it has no down-type variant; phase B does. - vk_pipeline pipeline_wp_fused_expert_batch_a; - vk_pipeline pipeline_wp_fused_expert_batch_b_q5_1; - vk_pipeline pipeline_wp_fused_expert_batch_b_q8_0; - - vk_pipeline pipeline_mul_mat_vec_p021_f16_f32[p021_max_gqa_ratio]; - vk_pipeline pipeline_mul_mat_vec_nc_f16_f32; - vk_pipeline pipeline_get_rows[GGML_TYPE_COUNT]; - vk_pipeline pipeline_get_rows_f32[GGML_TYPE_COUNT]; - vk_pipeline pipeline_get_rows_back_f32; - vk_pipeline pipeline_acc_f32; - vk_pipeline pipeline_set_f32; - - // [src0 0=fp32,1=fp16][src1 0=fp32,1=fp16][dst 0=fp32,1=fp16] - vk_pipeline pipeline_add[2][2][2]; - vk_pipeline pipeline_add_norepeat[2][2][2]; - vk_pipeline pipeline_sub[2][2][2]; - vk_pipeline pipeline_sub_norepeat[2][2][2]; - vk_pipeline pipeline_mul[2][2][2]; - vk_pipeline pipeline_mul_norepeat[2][2][2]; - vk_pipeline pipeline_div[2][2][2]; - vk_pipeline pipeline_div_norepeat[2][2][2]; - vk_pipeline pipeline_add_rms[2][2][2]; - vk_pipeline pipeline_add_rms_norepeat[2][2][2]; - - // indexed by num_additional_fused_ops == num_adds - 1 - vk_pipeline pipeline_multi_add[MAX_FUSED_ADDS]; - vk_pipeline pipeline_multi_add_rms[MAX_FUSED_ADDS]; - - vk_pipeline pipeline_add_id_f32; - - vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64; - vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32; - vk_pipeline pipeline_scale_f32; - vk_pipeline pipeline_log[2]; - vk_pipeline pipeline_tri[2]; - vk_pipeline pipeline_diag[2]; - vk_pipeline pipeline_clamp[2]; - vk_pipeline pipeline_pad_f32; - vk_pipeline pipeline_pad_reflect_1d_f32; - vk_pipeline pipeline_roll_f32; - vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32; - vk_pipeline pipeline_repeat_i16; - vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16, pipeline_cpy_f16_f32, pipeline_cpy_f32_bf16, pipeline_cpy_bf16_f32, pipeline_cpy_f32_i32, pipeline_cpy_i32_f32; - vk_pipeline pipeline_contig_cpy_f32_f32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_f16_f16, pipeline_contig_cpy_f16_f32, pipeline_contig_cpy_f32_bf16, pipeline_contig_cpy_bf16_f32, pipeline_contig_cpy_f32_i32, pipeline_contig_cpy_i32_f32; - vk_pipeline pipeline_cpy_f32_quant[GGML_TYPE_COUNT]; - vk_pipeline pipeline_cpy_quant_f32[GGML_TYPE_COUNT]; - vk_pipeline pipeline_cpy_transpose_16, pipeline_cpy_transpose_32; - vk_pipeline pipeline_cpy_transpose_02_16, pipeline_cpy_transpose_02_32; - // [src0 0=fp32,1=fp16][dst] - vk_pipeline pipeline_set_rows_i32[2][GGML_TYPE_COUNT]; - vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT]; - // SP2 paged attention (GGML_OP_PAGED_ATTN_MT), indexed by KV cache type. - vk_pipeline pipeline_paged_attn_scatter[GGML_TYPE_COUNT]; - vk_pipeline pipeline_paged_attn[GGML_TYPE_COUNT]; - // SP2 Task 5: split-K decode (q_len==1). Pass-1 per cache type, pass-2 reduce type-agnostic. - vk_pipeline pipeline_paged_attn_decode[GGML_TYPE_COUNT]; - vk_pipeline pipeline_paged_attn_decode_reduce; - vk_pipeline pipeline_norm_f32; - vk_pipeline pipeline_group_norm_f32; - vk_pipeline pipeline_turbo_wht_f32; - vk_pipeline pipeline_rms_norm_f32; - vk_pipeline pipeline_rms_norm_mul_f32; - vk_pipeline pipeline_rms_norm_partials_f32; - vk_pipeline pipeline_rms_norm_mul_partials_f32; - vk_pipeline pipeline_rms_norm_mul_rope_f32_f32; - vk_pipeline pipeline_rms_norm_mul_rope_f32_f16; - vk_pipeline pipeline_rms_norm_back_f32; - vk_pipeline pipeline_l2_norm_f32; - vk_pipeline pipeline_sinkhorn_norm_f32[3]; - - // [src/dst 0=fp32,1=fp16] - vk_pipeline pipeline_exp[2]; - vk_pipeline pipeline_expm1[2]; - vk_pipeline pipeline_elu[2]; - vk_pipeline pipeline_gelu[2]; - vk_pipeline pipeline_gelu_erf[2]; - vk_pipeline pipeline_gelu_quick[2]; - vk_pipeline pipeline_silu[2]; - vk_pipeline pipeline_relu[2]; - vk_pipeline pipeline_sqr[2]; - vk_pipeline pipeline_sqrt[2]; - vk_pipeline pipeline_sin[2]; - vk_pipeline pipeline_cos[2]; - vk_pipeline pipeline_xielu[2]; - vk_pipeline pipeline_neg[2]; - vk_pipeline pipeline_tanh[2]; - vk_pipeline pipeline_sigmoid[2]; - vk_pipeline pipeline_hardsigmoid[2]; - vk_pipeline pipeline_hardswish[2]; - vk_pipeline pipeline_abs[2]; - vk_pipeline pipeline_softplus[2]; - vk_pipeline pipeline_step[2]; - vk_pipeline pipeline_round[2]; - vk_pipeline pipeline_ceil[2]; - vk_pipeline pipeline_floor[2]; - vk_pipeline pipeline_trunc[2]; - vk_pipeline pipeline_sgn[2]; - - vk_pipeline pipeline_add1_f16_f16; - vk_pipeline pipeline_add1_f16_f32; - vk_pipeline pipeline_add1_f32_f32; - - vk_pipeline pipeline_arange_f32; - - vk_pipeline pipeline_fill_f32; - vk_pipeline pipeline_fill_f16; - - vk_pipeline pipeline_geglu[2]; - vk_pipeline pipeline_reglu[2]; - vk_pipeline pipeline_swiglu[2]; - vk_pipeline pipeline_swiglu_oai[2]; - vk_pipeline pipeline_swiglu_clamp[2]; - vk_pipeline pipeline_geglu_erf[2]; - vk_pipeline pipeline_geglu_quick[2]; - - vk_pipeline pipeline_leaky_relu[2]; - vk_pipeline pipeline_silu_back_f32; - vk_pipeline pipeline_diag_mask_inf_f32; - vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16; - vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512; - vk_pipeline pipeline_soft_max_back_f32; - - vk_pipeline pipeline_soft_max_large1_f32, pipeline_soft_max_large1_f32_f16; - vk_pipeline pipeline_soft_max_large2_f32, pipeline_soft_max_large2_f32_f16; - vk_pipeline pipeline_soft_max_large3_f32, pipeline_soft_max_large3_f32_f16; - - vk_pipeline pipeline_rope_norm_f32, pipeline_rope_norm_f16, pipeline_rope_norm_f32_f16; - vk_pipeline pipeline_rope_neox_f32, pipeline_rope_neox_f16, pipeline_rope_neox_f32_f16; - vk_pipeline pipeline_rope_multi_f32, pipeline_rope_multi_f16, pipeline_rope_multi_f32_f16; - vk_pipeline pipeline_rope_vision_f32, pipeline_rope_vision_f16; - vk_pipeline pipeline_argsort_f32[num_argsort_pipelines]; - vk_pipeline pipeline_argsort_large_f32[num_argsort_pipelines]; - vk_pipeline pipeline_topk_f32[num_topk_pipelines]; - vk_pipeline pipeline_topk_radix_f32; - vk_pipeline pipeline_topk_radix_qsa; // qwen4 QSA indexer fusion (f16 mask) - vk_pipeline pipeline_sum_rows_f32; - vk_pipeline pipeline_cross_entropy_loss_f32, pipeline_cross_entropy_loss_f32_wg512; - vk_pipeline pipeline_cross_entropy_loss_back_f32, pipeline_cross_entropy_loss_back_f32_wg512; - vk_pipeline pipeline_fwht_f32[4]; - vk_pipeline pipeline_cumsum_f32; - vk_pipeline pipeline_cumsum_small_f32; - vk_pipeline pipeline_cumsum_multipass1_f32; - vk_pipeline pipeline_cumsum_multipass2_f32; - vk_pipeline pipeline_argmax_f32; - vk_pipeline pipeline_count_equal_i32; - std::map<vk_solve_tri_pipeline_state, vk_pipeline> pipeline_solve_tri_f32; - vk_pipeline pipeline_im2col_f32, pipeline_im2col_f32_f16; - vk_pipeline pipeline_im2col_3d_f32, pipeline_im2col_3d_f32_f16; - vk_pipeline pipeline_timestep_embedding_f32; - vk_pipeline pipeline_conv_transpose_1d_f32; - vk_pipeline pipeline_col2im_1d_f32; - vk_pipeline pipeline_col2im_1d_f16; - vk_pipeline pipeline_col2im_1d_bf16; - vk_pipeline pipeline_out_prod_f32; - vk_pipeline pipeline_snake_f32; - vk_pipeline pipeline_snake_f16; - vk_pipeline pipeline_snake_bf16; - vk_pipeline pipeline_pool1d_f32; - vk_pipeline pipeline_pool2d_f32; - vk_pipeline pipeline_rwkv_wkv6_f32; - vk_pipeline pipeline_rwkv_wkv7_f32; - vk_pipeline pipeline_gated_linear_attn_f32; - vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT]; - // [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128 - vk_pipeline pipeline_gated_delta_net[4][2]; - // chunked (UT-transform) variant, [size_idx]; GDA only, null when unsupported - vk_pipeline pipeline_gated_delta_net_chunked[4]; - // columns covered by one chunked workgroup, [size_idx] (0 when the pipeline is null) - uint32_t gated_delta_net_chunked_cols[4] = {0, 0, 0, 0}; - vk_pipeline pipeline_ssm_scan_f32_d128; - vk_pipeline pipeline_ssm_scan_f32_d256; - vk_pipeline pipeline_ssm_conv_f32; - vk_pipeline pipeline_ssm_conv_silu_f32; - vk_pipeline pipeline_ssm_conv_bias_silu_f32; - vk_pipeline pipeline_opt_step_adamw_f32; - vk_pipeline pipeline_opt_step_sgd_f32; - std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f32[CONV_SHAPE_COUNT]; - std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f16_f32[CONV_SHAPE_COUNT]; - std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f32[CONV_SHAPE_COUNT]; - std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f16_f32[CONV_SHAPE_COUNT]; - std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f32[CONV_SHAPE_COUNT]; - std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f16_f32[CONV_SHAPE_COUNT]; - vk_pipeline pipeline_conv2d_dw_whcn_f32, pipeline_conv2d_dw_whcn_f16_f32; - vk_pipeline pipeline_conv2d_dw_cwhn_f32, pipeline_conv2d_dw_cwhn_f16_f32; - - std::map<vk_fa_pipeline_state, vk_pipeline> pipeline_flash_attn_f32_f16; - - std::map<std::pair<uint32_t, uint32_t>, vk_pipeline> pipeline_fa_mask_opt; - - vk_pipeline pipeline_flash_attn_split_k_reduce; - vk_pipeline pipeline_count_experts; - - // [2] is for whether to take n_experts from spec constant (0) or push constant (1) - vk_pipeline pipeline_topk_moe[num_topk_moe_pipelines][2]; - - std::vector<vk_pipeline_ref> all_pipelines; - - std::vector<std::tuple<void*, size_t, vk_buffer>> pinned_memory; - - vk::Fence fence; - vk_buffer sync_staging; - vk_buffer host_read_staging; - vk_buffer host_read_source; - size_t host_read_source_offset {}; - size_t host_read_source_size {}; - // Device-local scratch for the fused-expert batch fast path's Phase A - // output (silu(gate)*up per expert/token/hidden-unit), consumed by - // Phase B in the same command buffer. Grows on demand, never shrinks -- - // mirrors host_read_staging's ensure_*() pattern below. - vk_buffer wp_fused_batch_scratch; - - ggml_backend_buffer_type buffer_type; - - bool disable_fusion; - bool disable_host_visible_vidmem; - bool allow_sysmem_fallback; - bool disable_graph_optimize; - - std::unique_ptr<vk_memory_logger> memory_logger; - - ~vk_device_struct() { - VK_LOG_DEBUG("destroy device " << name); - - device.destroyFence(fence); - - ggml_vk_destroy_buffer(sync_staging); - ggml_vk_destroy_buffer(host_read_staging); - ggml_vk_destroy_buffer(wp_fused_batch_scratch); - host_read_source.reset(); - - if (compute_queue) compute_queue->cmd_pool.destroy(device); - if (transfer_queue) transfer_queue->cmd_pool.destroy(device); - - // Explicitly clear to ensure queues drop their shared_ptrs to handles - // before the Vulkan logical device instance is destroyed - compute_queue.reset(); - transfer_queue.reset(); - - for (auto& pipeline : all_pipelines) { - if (pipeline.expired()) { - continue; - } - - vk_pipeline pl = pipeline.lock(); - ggml_vk_destroy_pipeline(device, pl); - } - all_pipelines.clear(); - - device.destroyDescriptorSetLayout(dsl); - - device.destroy(); - } -}; - -void vk_command_pool::init(vk_device& device, vk_queue *q_) { - cmd_buffers.clear(); - q = q_; - - vk::CommandPoolCreateInfo command_pool_create_info( - vk::CommandPoolCreateFlags(VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT), - q->queue_family_index); - pool = device->device.createCommandPool(command_pool_create_info); -} - -void vk_command_pool::destroy(vk::Device& device) { - device.destroyCommandPool(pool); - pool = nullptr; - cmd_buffers.clear(); -} - -static void ggml_vk_print_device_fault_info(const vk_device& device) { +void ggml_vk_print_device_fault_info(const vk_device& device) { if (!device->device_fault || !device->pfn_vkGetDeviceFaultInfoEXT) { return; } @@ -1340,10217 +181,8761 @@ static void ggml_vk_print_device_fault_info(const vk_device& device) { (unsigned long long)info.vendorFaultData); } } - -struct vk_buffer_struct { - vk::Buffer buffer = VK_NULL_HANDLE; - vk::DeviceMemory device_memory = VK_NULL_HANDLE; - vk::MemoryPropertyFlags memory_property_flags; - void * ptr; - size_t size = 0; - vk::DeviceAddress bda_addr {}; - - vk_device device; - - ~vk_buffer_struct() { - if (size == 0) { - return; +uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) { + if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) { + const uint64_t m = node->ne[0]; + const uint64_t n = node->ne[1]; + const uint64_t k = node->src[1]->ne[0]; + const uint64_t batch = node->ne[2] * node->ne[3]; + return m * n * (k + (k - 1)) * batch; + } + if (node->op == GGML_OP_CONV_2D || node->op == GGML_OP_CONV_TRANSPOSE_2D) { + const ggml_tensor * knl = node->src[0]; + const uint64_t Cout = node->ne[2]; + const uint64_t size_K = node->src[1]->ne[2] * knl->ne[0] * knl->ne[1]; + const uint64_t size_N = node->ne[3] * node->ne[0] * node->ne[1]; + return Cout * size_N * (size_K + (size_K - 1)); + } + if (node->op == GGML_OP_CONV_3D) { + const ggml_tensor * knl = node->src[0]; + const uint64_t OC = ggml_get_op_params_i32(node, 11); + const uint64_t IC = ggml_get_op_params_i32(node, 9); + const uint64_t size_K = IC * knl->ne[0] * knl->ne[1] * knl->ne[2]; + const uint64_t size_N = node->ne[3] / OC * node->ne[0] * node->ne[1] * node->ne[2]; + return OC * size_N * (size_K + (size_K - 1)); + } + if (node->op == GGML_OP_FLASH_ATTN_EXT) { + const ggml_tensor * q = node->src[0]; + const ggml_tensor * k = node->src[1]; + const ggml_tensor * v = node->src[2]; + return 2ull * q->ne[1] * q->ne[2] * (k->ne[0] + v->ne[0]) * k->ne[1] * q->ne[3]; + } + return 0; +} +void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end) { + uint64_t total_flops = 0; + int n_ops = 0; + for (int j = start; j <= end && j < cgraph->n_nodes; j++) { + uint64_t flops = ggml_vk_get_node_flops(cgraph->nodes[j]); + total_flops += flops; + n_ops++; + if (flops > 0) { + GGML_LOG_CONT(" node %d: %s (%s) [%.2f GFLOP]\n", + j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op), + flops / 1e9); + } else { + GGML_LOG_CONT(" node %d: %s (%s)\n", + j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op)); } - VK_LOG_DEBUG("~vk_buffer_struct(" << buffer << ", " << size << ")"); - - device->device.freeMemory(device_memory); - device->device.destroyBuffer(buffer); } -}; + GGML_LOG_CONT(" total: %d ops, %.2f GFLOP\n", n_ops, total_flops / 1e9); +} +void ggml_vk_print_device_lost_info(const vk_device& device) { + ggml_vk_print_device_fault_info(device); + if (device->serialize_submissions && device->diag_cgraph != nullptr && device->diag_prev_start >= 0) { + GGML_LOG_ERROR("ggml_vulkan: device lost on %s, likely caused by previous submission (nodes %d to %d):\n", + device->name.c_str(), device->diag_prev_start, device->diag_prev_end); + ggml_vk_print_node_list(device->diag_cgraph, device->diag_prev_start, device->diag_prev_end); + } else { + GGML_LOG_ERROR("ggml_vulkan: device lost on %s\n", device->name.c_str()); + } +} +void * const vk_ptr_base = (void *)(uintptr_t) 0x1000; // NOLINT -struct vk_subbuffer { - vk_buffer buffer; - uint64_t offset; - uint64_t size; - const ggml_tensor * tensor = nullptr; - bool allow_misalign = false; +uint64_t vk_tensor_offset(const ggml_tensor * tensor) { + if (tensor->view_src) { + return (uint8_t *) tensor->view_src->data - (uint8_t *) vk_ptr_base; + } + return (uint8_t *) tensor->data - (uint8_t *) vk_ptr_base; +} - operator vk::DescriptorBufferInfo() const { - return { buffer->buffer, offset, size }; +size_t ggml_vk_tensor_buffer_offset(const ggml_backend_vk_context * ctx, const ggml_tensor * t) { + // vk_tensor_offset() is relative to vk_ptr_base, but mapped host tensors need an offset relative to their Vulkan buffer. + if (ctx->device->uma) { + vk_buffer buf = nullptr; + size_t off = 0; + ggml_vk_host_get(ctx->device, t->data, buf, off); + if (buf) { + return off; + } + } + return (size_t)(vk_tensor_offset(t) + t->view_offs); +} +size_t ggml_vk_descriptor_offset(size_t tensor_offset, size_t alignment, size_t type_size) { + // Move the descriptor back until its distance to the tensor is divisible by the tensor type size. + size_t descriptor_offset = tensor_offset & ~(alignment - 1); + while ((tensor_offset - descriptor_offset) % type_size != 0) { + GGML_ASSERT(descriptor_offset >= alignment); + descriptor_offset -= alignment; } -}; -struct vk_semaphore { - vk::Semaphore s; - uint64_t value; -}; + return descriptor_offset; +} +uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t) { + const size_t tensor_offset = ggml_vk_tensor_buffer_offset(ctx, t); + const size_t descriptor_offset = ggml_vk_descriptor_offset( + tensor_offset, ctx->device->properties.limits.minStorageBufferOffsetAlignment, ggml_type_size(t->type)); + GGML_ASSERT(tensor_offset - descriptor_offset <= UINT32_MAX); + return tensor_offset - descriptor_offset; +} -// vk_event is used for the event-related backend interfaces. It uses vk::Events for -// event_wait and a timeline semaphore for event_synchronize. Polling on an event for -// event_synchronize wouldn't be sufficient to wait for command buffers to complete, -// and would lead to validation errors. -struct vk_event { - std::vector<vk::Event> events_free; // Events available for reuse - std::vector<vk::Event> events_submitted; // Events that are fully submitted and can be reused on next synchronize - vk::Event event; - bool has_event; - - vk_semaphore tl_semaphore; - vk_command_buffer* cmd_buffer = nullptr; - uint64_t cmd_buffer_use_counter = 0; -}; +uint32_t ggml_vk_concat_unit_size(ggml_type type) { + const uint32_t type_size = ggml_type_size(type); -struct vk_submission { - vk_command_buffer* buffer = nullptr; - std::vector<vk_semaphore> wait_semaphores; - std::vector<vk_semaphore> signal_semaphores; -}; + if (!ggml_is_quantized(type)) { + return type_size; + } -typedef std::vector<vk_submission> vk_sequence; + // Use the widest existing concat shader that evenly divides a quant block. + if (type_size % 8 == 0) { + return 8; + } + if (type_size % 4 == 0) { + return 4; + } + if (type_size % 2 == 0) { + return 2; + } + return 1; +} +bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) { + if (src0->type != src1->type || src0->type != dst->type) { + return false; + } -struct vk_mat_mat_push_constants { - uint32_t M; uint32_t N; uint32_t K; - uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; - uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d; - uint32_t base_work_group_z; uint32_t num_batches; - uint32_t k_split; - uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3; - uint32_t padded_N; -}; - -#define MAT_VEC_FUSION_FLAGS_BIAS0 0x1 -#define MAT_VEC_FUSION_FLAGS_BIAS1 0x2 -#define MAT_VEC_FUSION_FLAGS_SCALE0 0x4 -#define MAT_VEC_FUSION_FLAGS_SCALE1 0x8 - -struct vk_mat_vec_push_constants { - uint32_t ncols; - uint32_t stride_a; - uint32_t stride_b; - uint32_t stride_d; - uint32_t batch_stride_a; - uint32_t batch_stride_b; - uint32_t batch_stride_d; - uint32_t fusion_flags; - uint32_t base_work_group_y; - uint32_t ne02; - uint32_t ne12; - uint32_t broadcast2; - uint32_t broadcast3; -}; - -struct vk_mat_vec_p021_push_constants { - uint32_t ncols_x; - uint32_t nrows_x; - uint32_t nchannels_x; - uint32_t nchannels_y; - uint32_t b_offset; - uint32_t d_offset; - uint32_t fusion_flags; -}; - -struct vk_mat_vec_nc_push_constants { - uint32_t ncols_x; - uint32_t nrows_x; - uint32_t row_stride_x; - uint32_t channel_stride_x; - uint32_t channel_stride_y; - uint32_t channel_x_divisor; - uint32_t ne12; - uint32_t b_offset; - uint32_t d_offset; - uint32_t nb03; - uint32_t nb13; - uint32_t nb23; - uint32_t fusion_flags; -}; - -struct vk_mat_mat_id_push_constants { - uint32_t M; uint32_t N; uint32_t K; - uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; - uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d; - uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11; - uint32_t n_experts; - uint32_t hoist_row_ids; - // Weight paging. paged=0 keeps stock uniform-stride addressing so the - // non-paged path is bit-identical. paged=1 means binding 0 is the pager's - // pool and binding 5 holds each expert's offset within it, in ELEMENTS. - uint32_t paged; -}; -struct vk_mat_vec_id_push_constants { - uint32_t ncols; - uint32_t stride_a; - uint32_t stride_b; - uint32_t stride_d; - uint32_t batch_stride_a; - uint32_t batch_stride_b; - uint32_t batch_stride_d; - uint32_t fusion_flags; - uint32_t nei0; - uint32_t ne11; - uint32_t expert_i1; - uint32_t nbi1; - // Weight paging. paged=0 keeps the stock uniform-stride addressing, so the - // non-paged path is bit-identical. paged=1 means binding 0 is the pager's - // pool and binding 6 holds each expert's block offset within it. - uint32_t paged; -}; - -struct vk_flash_attn_push_constants { - uint32_t N; - uint32_t KV; - - uint32_t ne1; - uint32_t ne2; - uint32_t ne3; - - uint32_t neq2; - uint32_t neq3; - uint32_t nek2; - uint32_t nek3; - uint32_t nev2; - uint32_t nev3; - uint32_t nem1; - uint32_t nem2; - uint32_t nem3; - - uint32_t nb01; - uint32_t nb02; - uint32_t nb03; - uint32_t nb11; - uint32_t nb12; - uint32_t nb13; - uint32_t nb21; - uint32_t nb22; - uint32_t nb23; - - float scale; - float max_bias; - float logit_softcap; - - uint32_t mask_n_head_log2; - float m0; - float m1; - - uint32_t gqa_ratio; - uint32_t split_kv; - uint32_t k_num; -}; -static_assert(sizeof(vk_flash_attn_push_constants) <= 128, "sizeof(vk_flash_attn_push_constants) must be <= 128"); - -struct vk_op_push_constants { - uint32_t KX; - uint32_t KY; - float param1; - float param2; - float param3; - float param4; -}; - -struct vk_wp_fused_expert_push_constants { - uint32_t expert_index; -}; - -// No `phase` field: phase A (gate/up -> hidden scratch) and phase B (down + -// weighted fold across experts) are now separate shaders/pipelines (see -// wp_fused_expert_batch_a.comp / wp_fused_expert_batch_b.comp) selected by -// which pipeline is dispatched, not by a runtime branch on a push constant. -// This struct is kept only as an unused placeholder so the generic -// ggml_vk_create_pipeline/ggml_vk_dispatch_pipeline plumbing (which threads a -// push_constant_size through every pipeline uniformly) needs no special case -// for a phase-less pipeline. -struct vk_wp_fused_expert_batch_push_constants { - uint32_t reserved; -}; - -struct vk_op_fwht_push_constants { - uint32_t n_rows; - uint32_t src_offset; - uint32_t dst_offset; - float scale; -}; - -struct vk_op_turbo_wht_push_constants { - uint32_t n_groups; // total groups = groups_per_head * n_heads - uint32_t group_size; // WHT group size (128, 64, or 32) - uint32_t direction; // 0 = forward, 1 = inverse - uint32_t has_scale; // 1 if InnerQ scale_inv buffer is valid, else 0 - uint32_t head_dim; // src->ne[0] - uint32_t groups_per_head; // head_dim / group_size - uint32_t src_offset; // misalignment offset (elements) for src buffer - uint32_t dst_offset; // misalignment offset (elements) for dst buffer - uint32_t scale_offset; // misalignment offset (elements) for scale buffer -}; - -struct vk_op_count_experts_push_constants { - uint32_t ne00; - uint32_t ne01; - uint32_t nb00; - uint32_t nb01; - uint32_t a_offset; - uint32_t n_experts; - uint32_t hoist_row_ids; - uint32_t ne00mp; - uint32_t ne00L; -}; - -struct vk_op_glu_push_constants { - uint32_t N; - uint32_t ne00; - uint32_t ne20; - uint32_t mode; // 0: default, 1: swapped, 2: split - float alpha; // for swiglu_oai - float limit; - uint32_t nb00; - uint32_t nb01; - uint32_t nb02; - uint32_t nb03; - uint32_t nb10; - uint32_t nb11; - uint32_t nb12; - uint32_t nb13; - uint32_t nb20; - uint32_t nb21; - uint32_t nb22; - uint32_t nb23; - uint32_t ne21; - uint32_t ne22; - uint32_t misalign_offsets; - uint32_t ne2_012mp; uint32_t ne2_012L; - uint32_t ne2_01mp; uint32_t ne2_01L; - uint32_t ne2_0mp; uint32_t ne2_0L; -}; -static_assert(sizeof(vk_op_glu_push_constants) <= 128, "sizeof(vk_op_glu_push_constants) must be <= 128"); - -struct vk_op_unary_push_constants { - uint32_t ne; - uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03; - uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13; - uint32_t misalign_offsets; - float param1; float param2; float param3; float param4; - uint32_t ne0_012mp; uint32_t ne0_01mp; uint32_t ne0_0mp; uint32_t ne0_Ls; - uint32_t ne1_012mp; uint32_t ne1_01mp; uint32_t ne1_0mp; uint32_t ne1_Ls; -}; -static_assert(sizeof(vk_op_unary_push_constants) <= 128, "sizeof(vk_op_unary_push_constants) must be <= 128"); - -static vk_op_unary_push_constants vk_op_unary_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst, int64_t ne = 0) { - GGML_ASSERT(ne != 0 || (ggml_nelements(src0) == ggml_nelements(dst))); - ne = ne != 0 ? ne : ggml_nelements(dst); - GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max()); - - vk_op_unary_push_constants p{}; - p.ne = (uint32_t)ne; - - size_t src0_tsize = ggml_type_size(src0->type); - p.ne00 = (uint32_t)src0->ne[0]; - p.ne01 = (uint32_t)src0->ne[1]; - p.ne02 = (uint32_t)src0->ne[2]; - p.ne03 = (uint32_t)src0->ne[3]; - p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize); - p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize); - p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize); - p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize); - - size_t dst_tsize = ggml_type_size(dst->type); - p.ne10 = (uint32_t)dst->ne[0]; - p.ne11 = (uint32_t)dst->ne[1]; - p.ne12 = (uint32_t)dst->ne[2]; - p.ne13 = (uint32_t)dst->ne[3]; - p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize); - p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize); - p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize); - p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize); - - return p; // offsets are initialized later in ggml_vk_op -} - -struct vk_op_pad_push_constants { - uint32_t ne; - uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03; - uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13; - uint32_t misalign_offsets; - uint32_t circular; - - uint32_t lp0; uint32_t rp0; - uint32_t lp1; uint32_t rp1; - uint32_t lp2; uint32_t rp2; - uint32_t lp3; uint32_t rp3; -}; - -static vk_op_pad_push_constants vk_op_pad_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst) { - int64_t ne = ggml_nelements(dst); - GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max()); - - vk_op_pad_push_constants p{}; - p.ne = (uint32_t)ne; - - size_t src0_tsize = ggml_type_size(src0->type); - p.ne00 = (uint32_t)src0->ne[0]; - p.ne01 = (uint32_t)src0->ne[1]; - p.ne02 = (uint32_t)src0->ne[2]; - p.ne03 = (uint32_t)src0->ne[3]; - p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize); - p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize); - p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize); - p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize); - - size_t dst_tsize = ggml_type_size(dst->type); - p.ne10 = (uint32_t)dst->ne[0]; - p.ne11 = (uint32_t)dst->ne[1]; - p.ne12 = (uint32_t)dst->ne[2]; - p.ne13 = (uint32_t)dst->ne[3]; - p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize); - p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize); - p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize); - p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize); - - p.lp0 = dst->op_params[0]; - p.rp0 = dst->op_params[1]; - p.lp1 = dst->op_params[2]; - p.rp1 = dst->op_params[3]; - p.lp2 = dst->op_params[4]; - p.rp2 = dst->op_params[5]; - p.lp3 = dst->op_params[6]; - p.rp3 = dst->op_params[7]; - p.circular = dst->op_params[8]; - - return p; // fastdiv values and offsets are initialized later in ggml_vk_op -} - -// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1. -// Precompute mp (m' in the paper) and L such that division -// can be computed using a multiply (high 32b of 64b result) -// and a shift: -// -// n/d = (mulhi(n, mp) + n) >> L; -static void init_fastdiv_values(uint32_t d, uint32_t &mp, uint32_t &L) -{ - // compute L = ceil(log2(d)); - L = 0; - while (L < 32 && (uint32_t{1} << L) < d) { - L++; + if (!ggml_is_quantized(src0->type)) { + const size_t type_size = ggml_type_size(src0->type); + return type_size == 1 || type_size == 2 || type_size == 4 || type_size == 8; } - mp = (uint32_t)((uint64_t{1} << 32) * ((uint64_t{1} << L) - d) / d + 1); -} - -static uint32_t pack_fastdiv_L(uint32_t L0, uint32_t L1, uint32_t L2) { - return L0 | (L1 << 8) | (L2 << 16); -} - -template <typename T> void init_pushconst_fastdiv(T &p) { - GGML_UNUSED(p); - static_assert(!std::is_const<T>::value, "unexpected type"); + // Quantized tensor rows are block-aligned when created. + return ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(src1) && ggml_is_contiguous_rows(dst); } +static bool vk_instance_initialized = false; -template <> void init_pushconst_fastdiv(vk_op_unary_push_constants &p) { - // Compute magic values to divide by these six numbers. - uint32_t ne0_012L; - uint32_t ne0_01L; - uint32_t ne0_0L; - uint32_t ne1_012L; - uint32_t ne1_01L; - uint32_t ne1_0L; - - init_fastdiv_values(p.ne02*p.ne01*p.ne00, p.ne0_012mp, ne0_012L); - init_fastdiv_values(p.ne01*p.ne00, p.ne0_01mp, ne0_01L); - init_fastdiv_values(p.ne00, p.ne0_0mp, ne0_0L); - init_fastdiv_values(p.ne12*p.ne11*p.ne10, p.ne1_012mp, ne1_012L); - init_fastdiv_values(p.ne11*p.ne10, p.ne1_01mp, ne1_01L); - init_fastdiv_values(p.ne10, p.ne1_0mp, ne1_0L); +vk_instance_t vk_instance; - p.ne0_Ls = pack_fastdiv_L(ne0_012L, ne0_01L, ne0_0L); - p.ne1_Ls = pack_fastdiv_L(ne1_012L, ne1_01L, ne1_0L); +static VkDeviceSize ggml_vk_get_max_buffer_range(const ggml_backend_vk_context * ctx, const vk_buffer &buf, const VkDeviceSize offset) { + const VkDeviceSize range = std::min(VkDeviceSize{buf->size - offset}, + VkDeviceSize{ctx->device->properties.limits.maxStorageBufferRange}); + return range; } -template <> void init_pushconst_fastdiv(vk_op_glu_push_constants &p) { - // GLU linearizes over dst, then uses dst coordinates for src0/src1. - init_fastdiv_values(p.ne22*p.ne21*p.ne20, p.ne2_012mp, p.ne2_012L); - init_fastdiv_values(p.ne21*p.ne20, p.ne2_01mp, p.ne2_01L); - init_fastdiv_values(p.ne20, p.ne2_0mp, p.ne2_0L); -} +void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) { + // Use waitForFences while most of the graph executes. Hopefully the CPU can sleep + // during this wait. + if (ctx->almost_ready_fence_pending) { + VK_CHECK(ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX), "almost_ready_fence", ctx->device); + ctx->device->device.resetFences({ ctx->almost_ready_fence }); + ctx->almost_ready_fence_pending = false; + } -template <> void init_pushconst_fastdiv(vk_op_count_experts_push_constants &p) { - init_fastdiv_values(p.ne00, p.ne00mp, p.ne00L); + // Spin (w/pause) waiting for the graph to finish executing. + vk::Result result; + for (;;) { + try { + result = ctx->device->device.getFenceStatus(ctx->fence); + } catch (vk::DeviceLostError &) { + ggml_vk_print_device_lost_info(ctx->device); + GGML_LOG_ERROR("ggml_vulkan: getFenceStatus at %s:%d\n", __FILE__, __LINE__); + throw; + } + if (result == vk::Result::eSuccess) { + break; + } + if (result != vk::Result::eNotReady) { + GGML_LOG_ERROR("ggml_vulkan: error %s at %s:%d\n", to_string(result).c_str(), __FILE__, __LINE__); + throw vk::SystemError(vk::make_error_code(result), "ggml_vulkan: getFenceStatus"); + } + for (uint32_t i = 0; i < 100; ++i) { + YIELD(); + YIELD(); + YIELD(); + YIELD(); + YIELD(); + YIELD(); + YIELD(); + YIELD(); + YIELD(); + YIELD(); + } + } + ctx->device->device.resetFences({ ctx->fence }); } -struct vk_op_binary_push_constants { - uint32_t ne; - uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03; - uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13; - uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; uint32_t nb20; uint32_t nb21; uint32_t nb22; uint32_t nb23; - uint32_t misalign_offsets; - float param1; float param2; int32_t param3; -}; - -// SP2 paged attention push constants. Layouts must match -// paged_attn_scatter.comp / paged_attn.comp respectively. -struct vk_op_paged_scatter_pc { - uint32_t HS; - uint32_t BS; - uint32_t n_kv_heads; - uint32_t n_tokens; - // Outlier-matrix sweep (2026-07-01, investigative-only): mirrors - // GGML_TURBO4_64_OL_TABLE read on the host (see - // ggml_vk_turbo4_64_ol_use_n64_table below / ggml_cuda_turbo4_64_ol_use_n64_table - // in turbo-quant.cuh). Nonzero selects TURBO_CENTROIDS_4BIT_N64 / - // TURBO_MID_4BIT_N64 for the turbo4_64_ol/ol8/ol12 non-outlier elements - // instead of the default TURBO_CENTROIDS_4BIT/TURBO_MID_4BIT table. - // Ignored by all other cache types. Not a permanent field. - uint32_t use_n64_table; -}; - -struct vk_op_paged_attn_pc { - uint32_t HS; - uint32_t BS; - uint32_t n_heads; - uint32_t n_kv_heads; - uint32_t max_blocks_per_seq; - uint32_t n_seq; - uint32_t n_tokens; - float scale; - // See vk_op_paged_scatter_pc::use_n64_table above (investigative-only). - uint32_t use_n64_table; -}; - -// SP2 Task 5 split-K decode push constants. Must match -// paged_attn_decode.comp / paged_attn_decode_reduce.comp respectively. -struct vk_op_paged_decode_pc { - uint32_t HS; - uint32_t BS; - uint32_t n_heads; - uint32_t n_kv_heads; - uint32_t max_blocks_per_seq; - uint32_t n_seq; - uint32_t num_splits; - float scale; - // See vk_op_paged_scatter_pc::use_n64_table above (investigative-only). - uint32_t use_n64_table; -}; - -struct vk_op_paged_decode_reduce_pc { - uint32_t HS; - uint32_t n_heads; - uint32_t n_seq; - uint32_t num_splits; -}; - -// Distinct type with the same layout so concat can overload tensor offset initialization. -struct vk_op_concat_push_constants : vk_op_binary_push_constants {}; -static_assert(sizeof(vk_op_concat_push_constants) == sizeof(vk_op_binary_push_constants)); -static_assert(std::is_standard_layout_v<vk_op_concat_push_constants>); +static bool ggml_vk_strip_decode_vector(const uint32_t * code, size_t word_count, std::vector<uint32_t> & out) { + static const char kDecodeVectorExt[] = "SPV_NV_cooperative_matrix_decode_vector"; -struct vk_op_multi_add_push_constants { - // shape for dst - uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; + if (word_count < 5) { + return false; + } - // strides for srcs+dst - uint32_t nb[MAX_PARAMETER_COUNT][4]; + bool uses_decode_vector = false; + for (size_t pos = 5; pos < word_count; ) { + uint32_t word = code[pos]; + uint32_t wc = word >> spv::WordCountShift; + uint32_t op = word & spv::OpCodeMask; + GGML_ASSERT(wc > 0 && pos + wc <= word_count); + if (op == spv::OpExtension && wc >= 2) { + const char * s = reinterpret_cast<const char *>(&code[pos + 1]); + if (strcmp(s, kDecodeVectorExt) == 0) { + uses_decode_vector = true; + break; + } + } + pos += wc; + } - uint32_t rms_partials; -}; -// update multi_add.comp if this changes -static_assert(MAX_PARAMETER_COUNT == 12); -static_assert(sizeof(vk_op_multi_add_push_constants) <= 256); - -struct vk_op_topk_moe_push_constants { - uint32_t n_rows; - uint32_t n_experts_push; - uint32_t n_expert_used; - float clamp_min; - float clamp_max; - uint32_t gating_func; - uint32_t has_bias; - uint32_t with_norm; - float output_scale; - float output_bias; -}; + if (!uses_decode_vector) { + return false; + } -struct vk_op_add_id_push_constants { - uint32_t ne0; - uint32_t ne1; - uint32_t s01; - uint32_t s02; - uint32_t s11; - uint32_t s21; -}; + VK_LOG_DEBUG("ggml_vk_strip_decode_vector: stripping SPV_NV_cooperative_matrix_decode_vector"); -struct vk_op_diag_mask_push_constants { - uint32_t ncols; - uint32_t rows_per_channel; - int32_t n_past; -}; + // Bulk-copy unchanged runs and only break the run when an instruction needs to + // be dropped or patched. Use reserve + insert/push_back so the destination buffer + // is touched exactly once (no zero-initialization pass from resize()). + out.clear(); + out.reserve(word_count); -struct vk_op_rope_push_constants { - uint32_t rope_mode; - uint32_t nrows; - uint32_t n_dims; - uint32_t n_offs; - float freq_scale; - float freq_base; - float ext_factor; - float attn_factor; - float corr_dims[2]; - float theta_scale; - uint32_t has_ff; - int32_t sections[4]; - uint32_t is_imrope; - uint32_t is_back; - uint32_t set_rows_stride; - uint32_t ne00; - uint32_t ne01; - uint32_t ne02; - uint32_t nb01; - uint32_t nb02; - uint32_t nb03; - uint32_t nb11; - uint32_t nb12; - uint32_t nb13; - uint32_t a_offset; - uint32_t d_offset; -}; -static_assert(sizeof(vk_op_rope_push_constants) <= 128, "sizeof(vk_op_rope_push_constants) must be <= 128"); + size_t run_start = 0; + auto flush_run = [&](size_t up_to) { + if (up_to > run_start) { + out.insert(out.end(), code + run_start, code + up_to); + } + }; -// For fused rms_norm+mul+rope(+view+set_rows) -struct vk_op_rms_norm_mul_rope_push_constants { - vk_op_binary_push_constants bin; - vk_op_rope_push_constants rope; -}; + for (size_t pos = 5; pos < word_count; ) { + uint32_t word = code[pos]; + uint32_t wc = word >> spv::WordCountShift; + uint32_t op = word & spv::OpCodeMask; + GGML_ASSERT(wc > 0 && pos + wc <= word_count); -struct vk_op_soft_max_push_constants { - uint32_t KX; - uint32_t KY; - uint32_t ne00; - uint32_t ne01; - uint32_t ne02; - uint32_t ne12; - uint32_t ne13; - uint32_t nb11; - uint32_t nb12; - uint32_t nb13; - float scale; - float max_bias; - float m0; - float m1; - uint32_t n_head_log2; - uint32_t nrows_x; - uint32_t has_sinks; -}; + if (op == spv::OpExtension && wc >= 2) { + const char * s = reinterpret_cast<const char *>(&code[pos + 1]); + if (strcmp(s, kDecodeVectorExt) == 0) { + flush_run(pos); + pos += wc; + run_start = pos; + continue; + } + } -struct vk_op_sinkhorn_norm_push_constants { - uint32_t n; - uint32_t nt; - float eps; - int32_t iters; - uint32_t a_offset; - uint32_t d_offset; -}; + if (op == spv::OpCapability && wc == 2 && code[pos + 1] == kSpvCapabilityCooperativeMatrixDecodeVectorNV) { + flush_run(pos); + pos += wc; + run_start = pos; + continue; + } -struct vk_op_argsort_push_constants { - uint32_t ncols; - uint32_t ncols_padded; - uint32_t ncols_padded_log2; - uint32_t nrows; - uint32_t order; - uint32_t outer_start; - uint32_t outer_end; - uint32_t inner_start; - uint32_t inner_end; -}; + if (op == kSpvOpCooperativeMatrixLoadTensorNV) { + // [opcode/wc][ResultType][Result][Pointer][Object][TensorLayout][MemOperand mask][mem extras...][TA mask][ta extras...] + GGML_ASSERT(wc >= 8); -struct vk_op_topk_push_constants { - uint32_t orig_ncols; - uint32_t ncols_input; - uint32_t ncols_output; - uint32_t k; - uint32_t nrows; - uint32_t first_pass; - uint32_t last_pass; -}; + uint32_t mem_mask = code[pos + 6]; + size_t cur = pos + 7; + // Each of these MemoryAccess bits (when set) carries one trailing operand. + cur += (mem_mask & 0x2) ? 1 : 0; // Aligned + cur += (mem_mask & 0x8) ? 1 : 0; // MakePointerAvailable + cur += (mem_mask & 0x10) ? 1 : 0; // MakePointerVisible + cur += (mem_mask & 0x10000) ? 1 : 0; // AliasScopeINTELMask + cur += (mem_mask & 0x20000) ? 1 : 0; // NoAliasINTELMask + GGML_ASSERT(cur < pos + wc); -struct vk_op_topk_radix_push_constants { - uint32_t ncols; - uint32_t k; - uint32_t nrows; - uint32_t n_tps; // QSA only - uint32_t n_blocks; // QSA only - uint32_t n_stream; // QSA only -}; + uint32_t ta_mask = code[cur]; + if ((ta_mask & kSpvTensorAddressingDecodeVectorFuncBit) == 0) { + pos += wc; + continue; // leave instruction inside the current unchanged run + } -struct vk_op_im2col_push_constants { - uint64_t dst_addr; - uint32_t batch_offset; uint32_t offset_delta; - uint32_t IC; - uint32_t IW; uint32_t IH; - uint32_t OW; uint32_t OH; - uint32_t KW; uint32_t KH; - uint32_t OH_batch; - uint32_t CHW; - int32_t s0; int32_t s1; - int32_t p0; int32_t p1; - int32_t d0; int32_t d1; - uint32_t batch_IC; -}; + flush_run(pos); -struct vk_op_im2col_3d_push_constants { - uint64_t dst_addr; - uint32_t nb10; - uint32_t nb11; - uint32_t nb12; - uint32_t nb13; - uint32_t s0; - uint32_t s1; - uint32_t s2; - uint32_t p0; - uint32_t p1; - uint32_t p2; - uint32_t d0; - uint32_t d1; - uint32_t d2; - uint32_t IW; - uint32_t IH; - uint32_t ID; - uint32_t IC; - uint32_t KW; - uint32_t OH; - uint32_t KD_KH_KW; - uint32_t KH_KW; - uint32_t IC_KD_KH_KW; - uint32_t N_OD_OH; - uint32_t OD_OH; - uint32_t OD_OH_OW_IC_KD_KH_KW; - uint32_t OH_OW_IC_KD_KH_KW; - uint32_t OW_IC_KD_KH_KW; - uint32_t misalign_offsets; -}; + // Append unchanged prefix of the instruction (header through the mem-extras). + size_t inst_start = out.size(); + size_t pre_n = cur - pos; + out.insert(out.end(), code + pos, code + pos + pre_n); -struct vk_op_timestep_embedding_push_constants { - uint32_t nb1; - uint32_t dim; - uint32_t max_period; -}; + // Emit TA mask with the DecodeVectorFunc bit cleared. + out.push_back(ta_mask & ~kSpvTensorAddressingDecodeVectorFuncBit); -struct vk_op_col2im_1d_push_constants { - uint32_t T_out; - uint32_t OC; - uint32_t K_OC; - uint32_t T_in; - uint32_t K; - int32_t stride; - int32_t p0; -}; + // TA extras: TensorView (0x1) and DecodeFunc (0x2) are kept verbatim; + // DecodeVectorFunc (0x4) is dropped along with its trailing id operand. + size_t keep_ta_extras = ((ta_mask & 0x1) ? 1 : 0) + ((ta_mask & 0x2) ? 1 : 0); + if (keep_ta_extras) { + out.insert(out.end(), code + cur + 1, code + cur + 1 + keep_ta_extras); + } -struct vk_op_conv_transpose_1d_push_constants { - uint32_t Cout; - uint32_t Cin; - uint32_t K; - uint32_t L; - uint32_t KL; + GGML_ASSERT(wc == pre_n + 1 + keep_ta_extras + 1); - uint32_t nb01; - uint32_t nb02; - uint32_t nb11; - uint32_t nb1; + // Patch the instruction header with the new (one-shorter) word count. + uint32_t new_wc = wc - 1; + out[inst_start] = (new_wc << spv::WordCountShift) | op; - int32_t s0; -}; + pos += wc; + run_start = pos; + continue; + } -struct vk_op_snake_push_constants { - uint32_t ne0; - uint32_t ne1; -}; + pos += wc; + } -struct vk_op_pool1d_push_constants { - uint32_t IL; - uint32_t OL; - uint32_t OC; - uint32_t pelements; - uint32_t op; - int32_t k0; - int32_t s0; - int32_t p0; -}; + flush_run(word_count); + return true; +} -struct vk_op_pool2d_push_constants { - uint32_t IW; uint32_t IH; - uint32_t OW; uint32_t OH; - uint32_t OC; - uint32_t pelements; - uint32_t op; - int32_t k0; int32_t k1; - int32_t s0; int32_t s1; - int32_t p0; int32_t p1; -}; +static bool ggml_vk_roll_bk_loop(const uint32_t * code, size_t word_count, std::vector<uint32_t> & out) { + if (word_count < 5) { + return false; + } -struct vk_op_rwkv_wkv6_push_constants { - uint32_t B; - uint32_t T; - uint32_t C; - uint32_t H; -}; + struct vk_spv_loop { + size_t header; + size_t end; + uint32_t control; + }; -struct vk_op_rwkv_wkv7_push_constants { - uint32_t B; - uint32_t T; - uint32_t C; - uint32_t H; -}; -struct vk_op_gated_linear_attn_push_constants { - uint32_t B; - uint32_t T; - uint32_t C; - uint32_t H; - float scale; -}; -struct vk_op_lightning_indexer_push_constants { - uint32_t n_kv; - uint32_t n_heads; - uint32_t n_tokens; - uint32_t n_streams; - uint32_t n_masks; - uint32_t dispatch_x; - uint32_t q_nb1; - uint32_t q_nb2; - uint32_t q_nb3; - uint32_t k_nb2; - uint32_t k_nb3; - uint32_t w_nb1; - uint32_t w_nb3; - uint32_t m_nb1; - uint32_t m_nb3; - uint32_t d_nb1; - uint32_t d_nb3; -}; -static_assert(sizeof(vk_op_lightning_indexer_push_constants) <= 128); -struct vk_op_gated_delta_net_push_constants { - uint32_t H; - uint32_t n_tokens; - uint32_t n_seqs; - uint32_t s_off; - uint32_t sq1, sq2, sq3; - uint32_t sv1, sv2, sv3; - uint32_t sb1, sb2, sb3; - uint32_t neq1, rq3; - float scale; - uint32_t K; -}; + std::vector<vk_spv_loop> loops; -struct vk_op_ssm_scan_push_constants { - uint32_t nb02, nb03, nb12, nb13; - uint32_t nb21, nb22, nb31; - uint32_t nb42, nb43, nb52, nb53; - uint32_t s_off; - uint32_t n_head, d_head, n_group, n_tok; - uint32_t n_seq, K; -}; -struct vk_op_ssm_conv_push_constants { - uint32_t nb01, nb02; - uint32_t nb11; - uint32_t dst_nb0, dst_nb1, dst_nb2; - uint32_t nc, ncs, nr, n_t, n_s; -}; + // Collect a list of all loops in the module. + for (size_t pos = 5; pos < word_count; ) { + const uint32_t wc = code[pos] >> spv::WordCountShift; + const uint32_t op = code[pos] & spv::OpCodeMask; + if (wc == 0 || pos + wc > word_count) { + return false; + } -struct vk_op_conv2d_push_constants { - uint32_t Cout; - uint32_t Cin; - uint32_t N; + if (op == spv::OpLoopMerge && wc >= 4) { loops.push_back({ pos, 0, code[pos + 3] }); } - uint32_t W; - uint32_t H; - uint32_t OW; - uint32_t OH; + if (op == spv::OpLabel && wc >= 2) { + for (auto & l : loops) { + if (l.end == 0 && code[l.header + 1] == code[pos + 1]) { l.end = pos; } + } + } - uint32_t nb01; - uint32_t nb02; - uint32_t nb03; + pos += wc; + } - uint32_t nb11; - uint32_t nb12; - uint32_t nb13; + auto encloses = [](const vk_spv_loop & a, const vk_spv_loop & b) { + return a.header < b.header && b.header < a.end; + }; - uint32_t nb1; - uint32_t nb2; - uint32_t nb3; - - // init_fastdiv_values constants for dividing by OW, OW*OH - uint32_t OWmp; uint32_t OWL; - uint32_t OWOHmp; uint32_t OWOHL; -}; + // Find the BK loop. + const vk_spv_loop * bk = nullptr; + for (const auto & h : loops) { + if (h.control != spv::LoopControlUnrollMask) { + continue; + } + const vk_spv_loop * parent = nullptr; + bool has_child = false; + for (const auto & g : loops) { + if (encloses(g, h) && (!parent || g.header > parent->header)) { + parent = &g; + } + if (encloses(h, g)) { + has_child = true; + } + } + // BK loop should be the last loop nested inside the loop with no hint + // and have at least one child loop. + if (parent && + parent->control == spv::LoopControlMaskNone && + has_child && + (!bk || h.header > bk->header)) { + bk = &h; + } + } + if (!bk) { + return false; + } -template <> void init_pushconst_fastdiv(vk_op_conv2d_push_constants &p) { - // Compute magic values to divide by OW, OW*OH - init_fastdiv_values(p.OW, p.OWmp, p.OWL); - init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL); + // set DontUnroll instead of Unroll + out.assign(code, code + word_count); + out[bk->header + 3] = spv::LoopControlDontUnrollMask; + return true; } -struct vk_op_conv3d_push_constants { - uint32_t OC; - uint32_t IC; - uint32_t N; - - uint32_t IW; - uint32_t IH; - uint32_t ID; - uint32_t OW; - uint32_t OH; - uint32_t OD; - - uint32_t nb01; - uint32_t nb02; - uint32_t nb03; +static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipeline, size_t spv_size, const void* spv_data, const std::string entrypoint, + uint32_t parameter_count, std::array<uint32_t, 3> wg_denoms, std::vector<uint32_t> specialization_constants, + bool disable_robustness, bool require_full_subgroups, uint32_t required_subgroup_size) { + VK_LOG_DEBUG("ggml_vk_create_pipeline(" << device->name << ", " << pipeline->name << ", " << entrypoint << ", " << parameter_count << + ", (" << wg_denoms[0] << "," << wg_denoms[1] << "," << wg_denoms[2] << "), specialization_constants, " << + disable_robustness << ", " << require_full_subgroups << ", " << required_subgroup_size << ")"); + GGML_ASSERT(parameter_count > 0); + GGML_ASSERT(parameter_count <= MAX_PARAMETER_COUNT); + GGML_ASSERT(wg_denoms[0] > 0 && wg_denoms[1] > 0 && wg_denoms[2] > 0); // NOLINT - uint32_t nb11; - uint32_t nb12; - uint32_t nb13; + vk::ShaderModuleCreateInfo shader_module_create_info({}, spv_size, reinterpret_cast<const uint32_t *>(spv_data)); - uint32_t nb1; - uint32_t nb2; - uint32_t nb3; + // Patch SPIR-V to enable supported FP16 float controls, avoiding the need + // for separate shader variants. + std::vector<uint32_t> spirv; + if (device->float_controls_rte_fp16 || device->float_controls_denorm_preserve_fp16) { + const uint32_t* spv_words = reinterpret_cast<const uint32_t *>(spv_data); + size_t word_count = spv_size / sizeof(uint32_t); + spirv.assign(spv_words, spv_words + word_count); - uint32_t OWmp; uint32_t OWL; - uint32_t OWOHmp; uint32_t OWOHL; - uint32_t OWOHODmp; uint32_t OWOHODL; -}; + // Find insertion points respecting SPIR-V layout order: + // Header(5) -> OpCapability -> OpExtension -> ... -> OpEntryPoint -> OpExecutionMode -> ... + size_t pos = 5; // skip header + size_t cap_insert_pos = pos; + size_t ext_insert_pos = pos; + size_t exec_insert_pos = pos; + uint32_t entry_point_id = 0; -template <> void init_pushconst_fastdiv(vk_op_conv3d_push_constants &p) { - init_fastdiv_values(p.OW, p.OWmp, p.OWL); - init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL); - init_fastdiv_values(p.OW*p.OH*p.OD, p.OWOHODmp, p.OWOHODL); -} - -struct vk_op_conv2d_dw_push_constants { - uint32_t ne; - uint32_t batches; - uint32_t channels; - uint32_t dst_w; - uint32_t dst_h; - uint32_t src_w; - uint32_t src_h; - uint32_t knl_w; - uint32_t knl_h; - int32_t stride_x; - int32_t stride_y; - int32_t pad_x; - int32_t pad_y; - int32_t dilation_x; - int32_t dilation_y; -}; + while (pos < spirv.size()) { + uint32_t opcode = spirv[pos] & spv::OpCodeMask; + uint32_t len = spirv[pos] >> spv::WordCountShift; + if (len == 0) break; -struct vk_op_upscale_push_constants { - uint32_t ne; uint32_t a_offset; uint32_t d_offset; - uint32_t ne00; uint32_t ne01; - uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03; - uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; - float sf0; float sf1; float sf2; float sf3; - float pixel_offset; -}; + if (opcode == spv::OpCapability) { + cap_insert_pos = pos + len; + ext_insert_pos = pos + len; + } else if (opcode == spv::OpExtension) { + ext_insert_pos = pos + len; + } else if (opcode == spv::OpEntryPoint) { + entry_point_id = spirv[pos + 2]; + exec_insert_pos = pos + len; + } else if (opcode == spv::OpExecutionMode || opcode == spv::OpExecutionModeId) { + exec_insert_pos = pos + len; + } else if (entry_point_id != 0) { + break; + } -struct vk_op_sum_rows_push_constants -{ - uint32_t n_cols; - uint32_t ne01, ne02; - uint32_t nb01, nb02, nb03; - uint32_t nb11, nb12, nb13; - float weight; - uint32_t misalign_offsets; - uint32_t ne0_12mp, ne0_12L; - uint32_t ne0_1mp, ne0_1L; -}; + pos += len; + } -static vk_op_sum_rows_push_constants vk_op_sum_rows_push_constants_init(const ggml_tensor * src, const ggml_tensor * dst, int64_t n_cols) { - uint32_t type_size = (uint32_t)ggml_type_size(src->type); - vk_op_sum_rows_push_constants p = {}; - p.n_cols = (uint32_t)n_cols; - p.ne01 = (uint32_t)src->ne[1]; - p.ne02 = (uint32_t)src->ne[2]; - p.nb01 = (uint32_t)src->nb[1] / type_size; - p.nb02 = (uint32_t)src->nb[2] / type_size; - p.nb03 = (uint32_t)src->nb[3] / type_size; - p.nb11 = (uint32_t)dst->nb[1] / type_size; - p.nb12 = (uint32_t)dst->nb[2] / type_size; - p.nb13 = (uint32_t)dst->nb[3] / type_size; - p.weight = 1.0f; - return p; -} - -template <> void init_pushconst_fastdiv(vk_op_sum_rows_push_constants &p) { - init_fastdiv_values(p.ne01*p.ne02, p.ne0_12mp, p.ne0_12L); - init_fastdiv_values(p.ne01, p.ne0_1mp, p.ne0_1L); -} - -struct vk_quantize_q8_1_push_constants { - uint32_t ne; - uint32_t num_blocks; -}; + // Insert from latest position first so earlier indices stay valid. -struct vk_op_flash_attn_split_k_reduce_push_constants { - uint32_t D; - uint32_t ne1; - uint32_t ne2; - uint32_t ne3; - uint32_t k_num; - uint32_t sinks; -}; + if (device->float_controls_rte_fp16) { + // OpExecutionMode %entrypoint RoundingModeRTE 16 + uint32_t exec_mode[] = { (4u << spv::WordCountShift) | spv::OpExecutionMode, entry_point_id, spv::ExecutionModeRoundingModeRTE, 16 }; + spirv.insert(spirv.begin() + exec_insert_pos, std::begin(exec_mode), std::end(exec_mode)); + } -struct vk_op_flash_attn_mask_opt_push_constants { - uint32_t nem0; - uint32_t nem1; - uint32_t nem2; - uint32_t nbm1; - uint32_t nbm2; - uint32_t nbm3; - uint32_t nbd1; - uint32_t nbd2; - uint32_t nbd3; -}; + if (device->float_controls_denorm_preserve_fp16) { + // OpExecutionMode %entrypoint DenormPreserve 16 + uint32_t exec_mode[] = { (4u << spv::WordCountShift) | spv::OpExecutionMode, entry_point_id, spv::ExecutionModeDenormPreserve, 16 }; + spirv.insert(spirv.begin() + exec_insert_pos, std::begin(exec_mode), std::end(exec_mode)); + } -// Allow pre-recording command buffers -struct vk_staging_memcpy { - vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {} + // OpExtension "SPV_KHR_float_controls" + const char ext_str[] = "SPV_KHR_float_controls"; + size_t ext_str_words = CEIL_DIV(sizeof(ext_str), sizeof(uint32_t)); + std::vector<uint32_t> extension(1 + ext_str_words, 0); + extension[0] = (uint32_t)((1 + ext_str_words) << spv::WordCountShift) | spv::OpExtension; + memcpy(&extension[1], ext_str, sizeof(ext_str)); + spirv.insert(spirv.begin() + ext_insert_pos, extension.begin(), extension.end()); - void * dst; - const void * src; - size_t n; -}; + if (device->float_controls_rte_fp16) { + // OpCapability RoundingModeRTE + uint32_t capability[] = { (2u << spv::WordCountShift) | spv::OpCapability, spv::CapabilityRoundingModeRTE }; + spirv.insert(spirv.begin() + cap_insert_pos, std::begin(capability), std::end(capability)); + } -struct vk_staging_memset { - vk_staging_memset(void * _dst, uint32_t _val, size_t _n) : dst(_dst), val(_val), n(_n) {} + if (device->float_controls_denorm_preserve_fp16) { + // OpCapability DenormPreserve + uint32_t capability[] = { (2u << spv::WordCountShift) | spv::OpCapability, spv::CapabilityDenormPreserve }; + spirv.insert(spirv.begin() + cap_insert_pos, std::begin(capability), std::end(capability)); + } - void * dst; - uint32_t val; - size_t n; -}; + shader_module_create_info = vk::ShaderModuleCreateInfo({}, spirv.size() * sizeof(uint32_t), spirv.data()); + } -struct vk_context_struct { - vk_submission * s; - vk_command_buffer * submitted_cmd_buf = nullptr; - std::vector<vk_sequence> seqs; +#if defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT) + if (device->coopmat2 && !device->coopmat2_decode_vector) { + const uint32_t * src = spirv.empty() ? reinterpret_cast<const uint32_t *>(spv_data) : spirv.data(); + size_t src_n = spirv.empty() ? spv_size / sizeof(uint32_t) : spirv.size(); + std::vector<uint32_t> stripped; + if (ggml_vk_strip_decode_vector(src, src_n, stripped)) { + spirv = std::move(stripped); + shader_module_create_info = vk::ShaderModuleCreateInfo({}, spirv.size() * sizeof(uint32_t), spirv.data()); + } + } +#endif - int exit_tensor_idx; +#if VK_HEADER_VERSION >= 287 + // Roll the mul_mm BK loop on Asahi Linux. Skip bf16 and the mul_mmq pipelines. + if (device->driver_id == vk::DriverId::eMesaHoneykrisp && + pipeline->name.rfind("matmul", 0) == 0 && + pipeline->name.find("bf16") == std::string::npos && + pipeline->name.find("q8_1") == std::string::npos) { + const uint32_t * src = spirv.empty() ? reinterpret_cast<const uint32_t *>(spv_data) : spirv.data(); + size_t src_n = spirv.empty() ? spv_size / sizeof(uint32_t) : spirv.size(); + std::vector<uint32_t> rolled; + if (ggml_vk_roll_bk_loop(src, src_n, rolled)) { + spirv = std::move(rolled); + shader_module_create_info = vk::ShaderModuleCreateInfo({}, spirv.size() * sizeof(uint32_t), spirv.data()); + } + } +#endif - std::vector<vk_staging_memcpy> in_memcpys; - std::vector<vk_staging_memcpy> out_memcpys; - std::vector<vk_staging_memset> memsets; + pipeline->shader_module = device->device.createShaderModule(shader_module_create_info); - // Pending contiguous same-type CPY regions, flushed as one vkCmdCopyBuffer. - vk_buffer cpy_src; - vk_buffer cpy_dst; - std::vector<vk::BufferCopy> cpy_regions; + vk::PushConstantRange pcr( + vk::ShaderStageFlagBits::eCompute, + 0, + pipeline->push_constant_size + ); - vk_command_pool * p {}; -}; -typedef std::shared_ptr<vk_context_struct> vk_context; -typedef std::weak_ptr<vk_context_struct> vk_context_ref; + vk::PipelineLayoutCreateInfo pipeline_layout_create_info(vk::PipelineLayoutCreateFlags(), device->dsl, pcr); + pipeline->layout = device->device.createPipelineLayout(pipeline_layout_create_info); -struct ggml_backend_vk_context; + std::vector<vk::SpecializationMapEntry> specialization_entries(specialization_constants.size()); -struct ggml_backend_vk_graph_plan { - struct descriptor_update { - vk::DescriptorSet descriptor_set; - std::vector<vk_subbuffer> sources; - }; + for (size_t i = 0; i < specialization_constants.size(); i++) { + specialization_entries[i].constantID = i; + specialization_entries[i].offset = i * sizeof(uint32_t); + specialization_entries[i].size = sizeof(uint32_t); + } - ggml_backend_vk_context * owner = nullptr; - vk_command_pool command_pool; - std::vector<vk_context> contexts; - std::vector<vk::DescriptorPool> descriptor_pools; - std::vector<vk::DescriptorSet> descriptor_sets; - std::vector<descriptor_update> descriptor_updates; - uint32_t descriptor_set_idx = 0; - uint32_t pipeline_descriptor_set_requirements = 0; - bool unsupported = false; -}; + vk::SpecializationInfo specialization_info( + specialization_entries.size(), + specialization_entries.data(), + specialization_constants.size() * sizeof(uint32_t), + specialization_constants.data() + ); -struct ggml_vk_garbage_collector { - std::vector<vk_semaphore> tl_semaphores; - std::vector<vk_semaphore> semaphores; - std::vector<vk::Event> events; - std::vector<vk_context> contexts; -}; + vk::PipelineShaderStageCreateFlags pipeline_shader_stage_create_flags{}; -static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx); -static void ggml_vk_flush_cpy_regions(vk_context& ctx); -static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr); -static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx); -static bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor); + if (device->subgroup_require_full_support && require_full_subgroups) { + pipeline_shader_stage_create_flags |= vk::PipelineShaderStageCreateFlagBits::eRequireFullSubgroupsEXT; + } -static bool vk_memory_logger_enabled = false; + vk::PipelineShaderStageCreateInfo pipeline_shader_create_info( + pipeline_shader_stage_create_flags, + vk::ShaderStageFlagBits::eCompute, + pipeline->shader_module, + entrypoint.c_str(), + &specialization_info); -#define VK_LOG_MEMORY(msg) if (vk_memory_logger_enabled) { std::cerr << "ggml_vulkan memory: " << msg << std::endl; } + vk::PipelineShaderStageRequiredSubgroupSizeCreateInfoEXT pipeline_shader_stage_required_subgroup_size_create_info; + pipeline_shader_stage_required_subgroup_size_create_info.requiredSubgroupSize = required_subgroup_size; + if (device->subgroup_size_control && required_subgroup_size > 0) { + GGML_ASSERT(device->subgroup_min_size <= required_subgroup_size && required_subgroup_size <= device->subgroup_max_size); + pipeline_shader_create_info.setPNext(&pipeline_shader_stage_required_subgroup_size_create_info); + } -static std::string format_size(size_t size) { - const size_t kib = 1024; - const size_t mib = kib * 1024; - const size_t gib = mib * 1024; + vk::ComputePipelineCreateInfo compute_pipeline_create_info( + device->pipeline_executable_properties_support ? + vk::PipelineCreateFlagBits::eCaptureStatisticsKHR : + vk::PipelineCreateFlags{}, + pipeline_shader_create_info, + pipeline->layout); - std::ostringstream oss; - oss << std::fixed << std::setprecision(2); + vk::PipelineRobustnessCreateInfoEXT rci; - if (size >= gib) { - oss << static_cast<double>(size) / gib << " GiB"; - } else if (size >= mib) { - oss << static_cast<double>(size) / mib << " MiB"; - } else if (size >= kib) { - oss << static_cast<double>(size) / kib << " KiB"; - } else { - oss << size << " B"; + if (device->pipeline_robustness && disable_robustness) { + rci.storageBuffers = vk::PipelineRobustnessBufferBehaviorEXT::eDisabled; + rci.uniformBuffers = vk::PipelineRobustnessBufferBehaviorEXT::eDisabled; + compute_pipeline_create_info.setPNext(&rci); } - return oss.str(); -} +#if defined(VK_EXT_shader_64bit_indexing) + vk::PipelineCreateFlags2CreateInfo pipelineFlags2CreateInfo; + if (pipeline->is_64b_indexing) + { + pipelineFlags2CreateInfo.flags = vk::PipelineCreateFlagBits2::e64BitIndexingEXT; + if (device->pipeline_executable_properties_support) { + pipelineFlags2CreateInfo.flags |= vk::PipelineCreateFlagBits2::eCaptureStatisticsKHR; + } + pipelineFlags2CreateInfo.setPNext(compute_pipeline_create_info.pNext); + compute_pipeline_create_info.setPNext(&pipelineFlags2CreateInfo); + } +#endif -class vk_memory_logger { -public: - vk_memory_logger(): total_device(0), total_host(0) {} - void log_allocation(vk_buffer_ref buf_ref, size_t size); - void log_deallocation(vk_buffer_ref buf_ref); + try { + pipeline->pipeline = device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value; + } catch (const vk::SystemError& e) { + std::cerr << "ggml_vulkan: Compute pipeline creation failed for " << pipeline->name << std::endl; + std::cerr << "ggml_vulkan: " << e.what() << std::endl; + throw e; + } -private: - std::map<vk::Buffer, size_t> allocations; // Track allocations - size_t total_device; - size_t total_host; - static std::mutex log_mutex; -}; + if (vk_instance.debug_utils_support) { + vk::DebugUtilsObjectNameInfoEXT duoni; + duoni.objectType = vk::ObjectType::ePipeline; + duoni.pObjectName = pipeline->name.c_str(); + duoni.objectHandle = /*reinterpret_cast*/(uint64_t)(static_cast<VkPipeline>(pipeline->pipeline)); + vk_instance.pfn_vkSetDebugUtilsObjectNameEXT(device->device, &static_cast<VkDebugUtilsObjectNameInfoEXT &>(duoni)); + } -std::mutex vk_memory_logger::log_mutex; + if (device->pipeline_executable_properties_support) { + vk::PipelineExecutableInfoKHR executableInfo; + executableInfo.pipeline = pipeline->pipeline; -static bool vk_perf_logger_enabled = false; -static bool vk_perf_logger_concurrent = false; -static bool vk_node_ts_enabled = false; -static bool vk_enable_sync_logger = false; + auto statistics = device->device.getPipelineExecutableStatisticsKHR(executableInfo); -static bool vk_want_timestamps() { - return vk_perf_logger_enabled || vk_node_ts_enabled; -} -// number of calls between perf logger prints -static uint32_t vk_perf_logger_frequency = 1; -static std::string vk_pipeline_stats_filter; + bool print_stats = !vk_pipeline_stats_filter.empty() && + pipeline->name.find(vk_pipeline_stats_filter) != std::string::npos; + if (print_stats) { + std::cerr << "ggml_vulkan: pipeline stats for " << pipeline->name << ":" << std::endl; + } -static uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) { - if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) { - const uint64_t m = node->ne[0]; - const uint64_t n = node->ne[1]; - const uint64_t k = node->src[1]->ne[0]; - const uint64_t batch = node->ne[2] * node->ne[3]; - return m * n * (k + (k - 1)) * batch; - } - if (node->op == GGML_OP_CONV_2D || node->op == GGML_OP_CONV_TRANSPOSE_2D) { - const ggml_tensor * knl = node->src[0]; - const uint64_t Cout = node->ne[2]; - const uint64_t size_K = node->src[1]->ne[2] * knl->ne[0] * knl->ne[1]; - const uint64_t size_N = node->ne[3] * node->ne[0] * node->ne[1]; - return Cout * size_N * (size_K + (size_K - 1)); - } - if (node->op == GGML_OP_CONV_3D) { - const ggml_tensor * knl = node->src[0]; - const uint64_t OC = ggml_get_op_params_i32(node, 11); - const uint64_t IC = ggml_get_op_params_i32(node, 9); - const uint64_t size_K = IC * knl->ne[0] * knl->ne[1] * knl->ne[2]; - const uint64_t size_N = node->ne[3] / OC * node->ne[0] * node->ne[1] * node->ne[2]; - return OC * size_N * (size_K + (size_K - 1)); + for (auto & s : statistics) { + if (print_stats) { + std::cerr << "ggml_vulkan: " << s.name.data() << ": "; + switch (s.format) { + case vk::PipelineExecutableStatisticFormatKHR::eBool32: + std::cerr << (s.value.b32 ? "true" : "false"); + break; + case vk::PipelineExecutableStatisticFormatKHR::eInt64: + std::cerr << s.value.i64; + break; + case vk::PipelineExecutableStatisticFormatKHR::eUint64: + std::cerr << s.value.u64; + break; + case vk::PipelineExecutableStatisticFormatKHR::eFloat64: + std::cerr << s.value.f64; + break; + } + std::cerr << std::endl; + } + // "Register Count" is reported by NVIDIA drivers. + if (strcmp(s.name, "Register Count") == 0) { + VK_LOG_DEBUG(pipeline->name << " " << s.name << ": " << s.value.u64 << " registers"); + pipeline->register_count = (uint32_t)s.value.u64; + } + } } - if (node->op == GGML_OP_FLASH_ATTN_EXT) { - const ggml_tensor * q = node->src[0]; - const ggml_tensor * k = node->src[1]; - const ggml_tensor * v = node->src[2]; - return 2ull * q->ne[1] * q->ne[2] * (k->ne[0] + v->ne[0]) * k->ne[1] * q->ne[3]; + + { + std::lock_guard<std::mutex> guard(device->compile_mutex); + device->all_pipelines.push_back(pipeline); + pipeline->compiled = true; + pipeline->compile_pending = false; } - return 0; + device->compile_cv.notify_all(); } -static void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end) { - uint64_t total_flops = 0; - int n_ops = 0; - for (int j = start; j <= end && j < cgraph->n_nodes; j++) { - uint64_t flops = ggml_vk_get_node_flops(cgraph->nodes[j]); - total_flops += flops; - n_ops++; - if (flops > 0) { - GGML_LOG_CONT(" node %d: %s (%s) [%.2f GFLOP]\n", - j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op), - flops / 1e9); - } else { - GGML_LOG_CONT(" node %d: %s (%s)\n", - j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op)); - } - } - GGML_LOG_CONT(" total: %d ops, %.2f GFLOP\n", n_ops, total_flops / 1e9); +void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline) { + VK_LOG_DEBUG("ggml_pipeline_destroy_pipeline(" << pipeline->name << ")"); + device.destroyPipelineLayout(pipeline->layout); + + device.destroyShaderModule(pipeline->shader_module); + + device.destroyPipeline(pipeline->pipeline); } -static void ggml_vk_print_device_lost_info(const vk_device& device) { - ggml_vk_print_device_fault_info(device); - if (device->serialize_submissions && device->diag_cgraph != nullptr && device->diag_prev_start >= 0) { - GGML_LOG_ERROR("ggml_vulkan: device lost on %s, likely caused by previous submission (nodes %d to %d):\n", - device->name.c_str(), device->diag_prev_start, device->diag_prev_end); - ggml_vk_print_node_list(device->diag_cgraph, device->diag_prev_start, device->diag_prev_end); - } else { - GGML_LOG_ERROR("ggml_vulkan: device lost on %s\n", device->name.c_str()); +void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, vk_pipeline& pipeline, uint32_t n) { + VK_LOG_DEBUG("ggml_pipeline_request_descriptor_sets(" << pipeline->name << ", " << n << ")"); + ggml_vk_pipeline_descriptor_set_requirements(ctx) += n; + if (!pipeline->compiled) { + ggml_vk_load_shaders(ctx->device, pipeline); } + ggml_pipeline_allocate_descriptor_sets(ctx); } -class vk_perf_logger { - public: - void print_timings(bool force = false) { - if (timings.empty()) { - return; - } - print_count++; - if ((print_count % vk_perf_logger_frequency) != 0 && !force) { - return; - } - print_count = 0; - uint64_t total_all_op_times = 0; - std::cerr << "----------------\nVulkan Timings:" << std::endl; - for (const auto & t : timings) { - uint64_t total_op_times = 0; - for (const auto & time : t.second) { - total_op_times += time; - } - std::cerr << t.first << ": " << t.second.size() << " x " << (total_op_times / t.second.size() / 1000.0) - << " us = " << (total_op_times / 1000.0) << " us"; - - // If we have as many flops entries as timing entries for the op, then compute and log the flops/S. - auto it = flops.find(t.first); - if (it != flops.end() && (it->second).size() == t.second.size()) { - uint64_t total_op_flops = 0; - for (const auto & elem : it->second) { - total_op_flops += elem; - } - std::cerr << " (" - << (double(total_op_flops) / (1000.0 * 1000.0 * 1000.0)) / - (double(total_op_times) / (1000.0 * 1000.0 * 1000.0)) - << " GFLOPS/s)"; - } +void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx) { + std::vector<vk::DescriptorPool> & descriptor_pools = ggml_vk_descriptor_pools(ctx); + std::vector<vk::DescriptorSet> & descriptor_sets = ggml_vk_descriptor_sets(ctx); + const uint32_t descriptor_set_requirements = + ggml_vk_pipeline_descriptor_set_requirements(ctx); + + if (descriptor_sets.size() >= descriptor_set_requirements) { + // Enough descriptors are available + return; + } - total_all_op_times += total_op_times; + vk_device& device = ctx->device; - std::cerr << std::endl; + // Grow by 50% to avoid frequent allocations + uint32_t needed = std::max(3 * descriptor_sets.size() / 2, size_t{descriptor_set_requirements}); + uint32_t to_alloc = needed - descriptor_sets.size(); + uint32_t pool_remaining = VK_DEVICE_DESCRIPTOR_POOL_SIZE - descriptor_sets.size() % VK_DEVICE_DESCRIPTOR_POOL_SIZE; + uint32_t pool_idx = descriptor_sets.size() / VK_DEVICE_DESCRIPTOR_POOL_SIZE; + + while (to_alloc > 0) { + const uint32_t alloc_count = std::min(pool_remaining, to_alloc); + to_alloc -= alloc_count; + pool_remaining = VK_DEVICE_DESCRIPTOR_POOL_SIZE; + + if (pool_idx >= descriptor_pools.size()) { + vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, MAX_PARAMETER_COUNT * VK_DEVICE_DESCRIPTOR_POOL_SIZE); + vk::DescriptorPoolCreateInfo descriptor_pool_create_info({}, VK_DEVICE_DESCRIPTOR_POOL_SIZE, descriptor_pool_size); + descriptor_pools.push_back(device->device.createDescriptorPool(descriptor_pool_create_info)); } - if (timings.size() > 0) { - std::cerr << "Total time: " << total_all_op_times / 1000.0 << " us." << std::endl; + std::vector<vk::DescriptorSetLayout> layouts(alloc_count); + for (uint32_t i = 0; i < alloc_count; i++) { + layouts[i] = device->dsl; } + vk::DescriptorSetAllocateInfo descriptor_set_alloc_info(descriptor_pools[pool_idx], alloc_count, layouts.data()); + std::vector<vk::DescriptorSet> sets = device->device.allocateDescriptorSets(descriptor_set_alloc_info); + descriptor_sets.insert(descriptor_sets.end(), sets.begin(), sets.end()); - timings.clear(); - flops.clear(); + pool_idx++; } +} - std::string get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops) { - *n_flops = ggml_vk_get_node_flops(node); - std::string fusion_str; - if (fusion_name) { - fusion_str = fusion_name + std::string(" "); - } - if (node->op == GGML_OP_UNARY) { - return fusion_str + ggml_unary_op_name(ggml_get_unary_op(node)); - } - if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) { - const uint64_t m = node->ne[0]; - const uint64_t n = node->ne[1]; - const uint64_t k = node->src[1]->ne[0]; - const uint64_t batch = node->ne[2] * node->ne[3]; - std::string name = ggml_op_name(node->op); - // Naming heuristic only; the perf logger has no device handle, so it - // uses the stock cap rather than this device's effective one. - if ((node->op == GGML_OP_MUL_MAT && n <= mul_mat_vec_max_cols_default) || - (node->op == GGML_OP_MUL_MAT_ID && node->src[2]->ne[1] == 1)) { - name += "_VEC"; - } - name += " "; - name += ggml_type_name(node->src[0]->type); - name += " m=" + std::to_string(m) + " n=" + std::to_string(n) + " k=" + std::to_string(k); - if (node->op == GGML_OP_MUL_MAT_ID) { - name += " n_expert=" + std::to_string(node->src[0]->ne[2]); - } - if (batch > 1) { - name += " batch=" + std::to_string(batch); - } - return fusion_str + name; - } - if (node->op == GGML_OP_CONV_2D || node->op == GGML_OP_CONV_TRANSPOSE_2D) { - std::string name = ggml_op_name(node->op); - const ggml_tensor * knl = node->src[0]; - uint64_t Cout = node->ne[2]; - uint64_t size_K = node->src[1]->ne[2] * knl->ne[0] * knl->ne[1]; - uint64_t size_N = node->ne[3] * node->ne[0] * node->ne[1]; - name += " M=Cout=" + std::to_string(Cout) + ", K=Cin*KW*KH=" + std::to_string(size_K) + - ", N=N*OW*OH=" + std::to_string(size_N); - return fusion_str + name; - } - if (node->op == GGML_OP_RMS_NORM) { - std::string name = ggml_op_name(node->op); - name += "(" + std::to_string(node->ne[0]) + "," + std::to_string(node->ne[1]) + "," + std::to_string(node->ne[2]) + "," + std::to_string(node->ne[3]) + ")"; - return fusion_str + name; - } - if (node->op == GGML_OP_FLASH_ATTN_EXT) { - const ggml_tensor * dst = node; - const ggml_tensor * q = node->src[0]; - const ggml_tensor * k = node->src[1]; - const ggml_tensor * v = node->src[2]; - const ggml_tensor * m = node->src[3]; - std::stringstream name; - name << fusion_str; - name << ggml_op_name(node->op) << - " dst(" << dst->ne[0] << "," << dst->ne[1] << "," << dst->ne[2] << "," << dst->ne[3] << "), " << - " q(" << q->ne[0] << "," << q->ne[1] << "," << q->ne[2] << "," << q->ne[3] << "), " << - " k(" << k->ne[0] << "," << k->ne[1] << "," << k->ne[2] << "," << k->ne[3] << "), " << - " v(" << v->ne[0] << "," << v->ne[1] << "," << v->ne[2] << "," << v->ne[3] << "), " << - " m(" << (m?m->ne[0]:0) << "," << (m?m->ne[1]:0) << "," << (m?m->ne[2]:0) << "," << (m?m->ne[3]:0) << ")"; - return name.str(); - } - if (node->op == GGML_OP_TOP_K) { - std::stringstream name; - name << fusion_str; - name << ggml_op_name(node->op) << - " K=" << node->ne[0] << - " (" << node->src[0]->ne[0] << "," << node->src[0]->ne[1] << "," << node->src[0]->ne[2] << "," << node->src[0]->ne[3] << ")"; - return name.str(); - } - return fusion_str + ggml_op_name(node->op); - } - - void log_timing(const ggml_tensor * node, const char *fusion_name, uint64_t time) { - uint64_t n_flops; - std::string name = get_node_fusion_name(node, fusion_name, &n_flops); - if (n_flops) { - flops[name].push_back(n_flops); - } - timings[name].push_back(time); - } - - void log_timing(const std::vector<ggml_tensor *> &nodes, const std::vector<const char *> &names, uint64_t time) { - uint64_t total_flops = 0; - std::string name; - for (size_t n = 0; n < nodes.size(); ++n) { - uint64_t n_flops = 0; - name += get_node_fusion_name(nodes[n], names[n], &n_flops); - total_flops += n_flops; - - if (n != nodes.size() - 1) { - name += ", "; - } - } - if (total_flops) { - flops[name].push_back(total_flops); +static vk_command_buffer* ggml_vk_create_cmd_buffer(vk_device& device, vk_command_pool& p) { + VK_LOG_DEBUG("ggml_vk_create_cmd_buffer()"); + vk::CommandBufferAllocateInfo command_buffer_alloc_info( + p.pool, + vk::CommandBufferLevel::ePrimary, + 1); + const std::vector<vk::CommandBuffer> cmd_buffers = device->device.allocateCommandBuffers(command_buffer_alloc_info); + p.cmd_buffers.push_back({ cmd_buffers.front(), 0, true }); + return &p.cmd_buffers[p.cmd_buffers.size()-1]; +} + +void ggml_vk_submit(vk_context& ctx, vk::Fence fence) { + if (ctx->seqs.empty()) { + if (fence) { + ctx->p->q->handle->submit({}, fence); } - timings[name].push_back(time); + return; } + VK_LOG_DEBUG("ggml_vk_submit(" << ctx << ", " << fence << ")"); - private: - std::map<std::string, std::vector<uint64_t>> timings; - std::map<std::string, std::vector<uint64_t>> flops; - uint32_t print_count {}; -}; - -struct ggml_backend_vk_context { - std::string name; + std::vector<std::vector<uint64_t>> tl_wait_vals; + std::vector<std::vector<uint64_t>> tl_signal_vals; + std::vector<std::vector<vk::Semaphore>> tl_wait_semaphores; + std::vector<std::vector<vk::Semaphore>> tl_signal_semaphores; + std::vector<vk::TimelineSemaphoreSubmitInfo> tl_submit_infos; + std::vector<vk::SubmitInfo> submit_infos; + int idx = -1; + std::vector<std::vector<vk::PipelineStageFlags>> stage_flags; - vk_device device; + size_t reserve = 0; - size_t semaphore_idx, event_idx; - ggml_vk_garbage_collector gc; - size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k, prealloc_size_add_rms_partials, prealloc_size_add_rms_partials_offset; - vk_buffer prealloc_x, prealloc_y, prealloc_split_k, prealloc_add_rms_partials, sync_staging; - // Per-expert block offsets for weight paging. One slice per mul_mat_id node - // because dispatches execute after recording; cursor resets each graph. - vk_buffer prealloc_wp_expert_off; - size_t wp_expert_off_cursor = 0; - std::vector<vk_buffer> wp_expert_off_retired; - vk::Fence fence, almost_ready_fence; - bool submit_pending {}; - bool almost_ready_fence_pending {}; - // WP_VK_FENCE_ON_LAST_SUBMIT: true when ctx->fence has a pending signal that - // was attached to the graph's own final submit rather than to a trailing - // fence-only vkQueueSubmit. Invariant: while it is true, NOTHING has been - // submitted on this context since -- every submitter drains it first. - bool fence_submitted {}; - // Compute-queue submit_epoch as of the submission ctx->fence rides on, so a - // later drain publishes coverage for exactly that much and no more. - uint64_t fence_epoch {}; - // Set before op_add and unset after op_rms_norm to indicate that the add should - // write partial sums to accumulate the square of the vector components - bool do_add_rms_partials_offset_calculation; - bool do_add_rms_partials; - - uint64_t last_total_flops {UINT64_MAX}; - - // Cache most recent tensor that was converted into prealloc_y, and what pipeline it used to convert. - vk_pipeline_struct * prealloc_y_last_pipeline_used {}; - const ggml_tensor * prealloc_y_last_tensor_used {}; - // True when the K dimension in prealloc_y is padded. - bool prealloc_y_last_k_padded {}; - - // Track which nodes have been used since the last sync, and whether they were written to - std::vector<const ggml_tensor *> unsynced_nodes_written; - std::vector<const ggml_tensor *> unsynced_nodes_read; - // Track which prealloc buffers have pending reads that need to be synchronized. - // These are checked before writing to the buffer (and call ggml_vk_sync_buffers if set), - // and set to true after the buffer contents are consumed. - bool prealloc_x_need_sync, prealloc_y_need_sync, prealloc_split_k_need_sync; - - vk_context_ref compute_ctx; - - // Non-null only while a persistent graph plan is being recorded. - ggml_backend_vk_graph_plan * recording_plan = nullptr; - - vk_context_ref transfer_ctx; - vk_semaphore transfer_semaphore; - uint64_t transfer_semaphore_last_submitted {}; - - std::vector<vk_context_ref> tensor_ctxs; - - std::vector<vk::DescriptorPool> descriptor_pools; - std::vector<vk::DescriptorSet> descriptor_sets; - std::array<vk::DescriptorBufferInfo, MAX_PARAMETER_COUNT> descriptor_buffer_infos{}; - uint32_t descriptor_set_idx {}; - uint32_t pipeline_descriptor_set_requirements {}; - - vk_command_pool compute_cmd_pool; - vk_command_pool transfer_cmd_pool; - - // number of additional consecutive nodes that are being fused with the - // node currently being processed - int num_additional_fused_ops {}; - // Bitmask of which fused ops need to write an intermediate value to memory. - // Bit 'i' means nodes[start_of_fusion + i] writes to memory. - // If there's no fusion, bit 0 is still set. - int fused_ops_write_mask {}; - topk_moe_mode fused_topk_moe_mode {}; - bool fused_topk_moe_scale {}; - // QSA indexer gather+add+top_k fused into one radix-select - bool fused_topk_qsa {}; - - // for GGML_VK_PERF_LOGGER and WP_VK_NODE_TS - std::unique_ptr<vk_perf_logger> perf_logger; - vk::QueryPool query_pool; - std::vector<const char *> query_fusion_names; - std::vector<int> query_fusion_node_count; - std::vector<ggml_tensor *> query_nodes; - std::vector<int> query_node_idx; - std::vector<ggml_op> query_ops; - int32_t num_queries {}; - int32_t query_idx {}; - bool node_ts_pending {}; - ggml_backend_vk_node_ts node_ts {}; -}; + for (const auto& sequence : ctx->seqs) { + reserve += sequence.size(); + } -static std::vector<vk::DescriptorPool> & ggml_vk_descriptor_pools(ggml_backend_vk_context * ctx) { - return ctx->recording_plan != nullptr ? ctx->recording_plan->descriptor_pools : ctx->descriptor_pools; -} + // Pre-reserve vectors to prevent reallocation, which invalidates pointers + tl_wait_semaphores.reserve(reserve); + tl_wait_vals.reserve(reserve); + tl_signal_semaphores.reserve(reserve); + tl_signal_vals.reserve(reserve); + tl_submit_infos.reserve(reserve); + submit_infos.reserve(reserve); + stage_flags.reserve(reserve); -static std::vector<vk::DescriptorSet> & ggml_vk_descriptor_sets(ggml_backend_vk_context * ctx) { - return ctx->recording_plan != nullptr ? ctx->recording_plan->descriptor_sets : ctx->descriptor_sets; -} + for (const auto& sequence : ctx->seqs) { + for (const auto& submission : sequence) { + stage_flags.push_back({}); + idx++; + tl_wait_vals.push_back({}); + tl_wait_semaphores.push_back({}); + tl_signal_vals.push_back({}); + tl_signal_semaphores.push_back({}); + for (size_t i = 0; i < submission.wait_semaphores.size(); i++) { + stage_flags[idx].push_back(ctx->p->q->stage_flags); + tl_wait_vals[idx].push_back(submission.wait_semaphores[i].value); + tl_wait_semaphores[idx].push_back(submission.wait_semaphores[i].s); + } + for (size_t i = 0; i < submission.signal_semaphores.size(); i++) { + tl_signal_vals[idx].push_back(submission.signal_semaphores[i].value); + tl_signal_semaphores[idx].push_back(submission.signal_semaphores[i].s); + } + tl_submit_infos.push_back({ + (uint32_t) submission.wait_semaphores.size(), + tl_wait_vals[idx].data(), + (uint32_t) submission.signal_semaphores.size(), + tl_signal_vals[idx].data(), + }); + tl_submit_infos[idx].sType = vk::StructureType::eTimelineSemaphoreSubmitInfo; + tl_submit_infos[idx].pNext = nullptr; + vk::SubmitInfo si{ + (uint32_t) submission.wait_semaphores.size(), + tl_wait_semaphores[idx].data(), + stage_flags[idx].data(), + 1, + &submission.buffer->buf, + (uint32_t) submission.signal_semaphores.size(), + tl_signal_semaphores[idx].data(), + }; + si.setPNext(&tl_submit_infos[idx]); + submit_infos.push_back(si); + } + } -static uint32_t & ggml_vk_descriptor_set_idx(ggml_backend_vk_context * ctx) { - return ctx->recording_plan != nullptr ? ctx->recording_plan->descriptor_set_idx : ctx->descriptor_set_idx; -} + ctx->p->q->handle->submit(submit_infos, fence); -static uint32_t & ggml_vk_pipeline_descriptor_set_requirements(ggml_backend_vk_context * ctx) { - return ctx->recording_plan != nullptr ? ctx->recording_plan->pipeline_descriptor_set_requirements : ctx->pipeline_descriptor_set_requirements; + ctx->seqs.clear(); } -static void * const vk_ptr_base = (void *)(uintptr_t) 0x1000; // NOLINT +uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyProperties>& queue_family_props, const vk::QueueFlags& required, const vk::QueueFlags& avoid, int32_t compute_index, uint32_t min_num_queues) { + VK_LOG_DEBUG("ggml_vk_find_queue_family_index()"); + const uint32_t qfsize = queue_family_props.size(); -static uint64_t vk_tensor_offset(const ggml_tensor * tensor) { - if (tensor->view_src) { - return (uint8_t *) tensor->view_src->data - (uint8_t *) vk_ptr_base; + // Try with avoid preferences first + for (uint32_t i = 0; i < qfsize; i++) { + if (queue_family_props[i].queueCount >= min_num_queues && (compute_index < 0 || i != (uint32_t) compute_index) && queue_family_props[i].queueFlags & required && !(queue_family_props[i].queueFlags & avoid)) { + return i; + } } - return (uint8_t *) tensor->data - (uint8_t *) vk_ptr_base; -} - -static uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t) -{ - return ((vk_tensor_offset(t) + t->view_offs) & (ctx->device->properties.limits.minStorageBufferOffsetAlignment - 1));; -} -static uint32_t ggml_vk_concat_unit_size(ggml_type type) { - const uint32_t type_size = ggml_type_size(type); + // Fall back to only required + for (size_t i = 0; i < qfsize; i++) { + if (queue_family_props[i].queueCount >= min_num_queues && (compute_index < 0 || i != (uint32_t) compute_index) && queue_family_props[i].queueFlags & required) { + return i; + } + } - if (!ggml_is_quantized(type)) { - return type_size; + // Fall back to reusing compute queue + for (size_t i = 0; i < qfsize; i++) { + if (queue_family_props[i].queueCount >= min_num_queues && queue_family_props[i].queueFlags & required) { + return i; + } } - // Use the widest existing concat shader that evenly divides a quant block. - if (type_size % 8 == 0) { - return 8; + // Fall back to ignoring min_num_queries + for (size_t i = 0; i < qfsize; i++) { + if (queue_family_props[i].queueFlags & required) { + return i; + } } - if (type_size % 4 == 0) { - return 4; + + // All commands that are allowed on a queue that supports transfer operations are also allowed on a queue that supports either graphics or compute operations. + // Thus, if the capabilities of a queue family include VK_QUEUE_GRAPHICS_BIT or VK_QUEUE_COMPUTE_BIT, then reporting the VK_QUEUE_TRANSFER_BIT capability separately for that queue family is optional. + if (compute_index >= 0) { + return compute_index; } - if (type_size % 2 == 0) { - return 2; + + std::cerr << "ggml_vulkan: No suitable queue family index found." << std::endl; + + for(auto &q_family : queue_family_props) { + std::cerr << "Queue number: " + std::to_string(q_family.queueCount) << " flags: " + to_string(q_family.queueFlags) << std::endl; } - return 1; + abort(); } -static bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) { - if (src0->type != src1->type || src0->type != dst->type) { - return false; +std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_t queue_family_index, uint32_t queue_index, vk::PipelineStageFlags&& stage_flags, bool transfer_only) { + VK_LOG_DEBUG("ggml_vk_create_queue()"); + std::lock_guard<std::recursive_mutex> guard(device->mutex); + + auto q = std::make_unique<vk_queue>(); + q->queue_family_index = queue_family_index; + q->transfer_only = transfer_only; + + std::shared_ptr<vk_queue_handle> h; + vk::DeviceQueueInfo2 queue_info2{}; + queue_info2.queueFamilyIndex = queue_family_index; + queue_info2.queueIndex = queue_index; + + if (device->has_internally_synchronized_queues) { + h = std::make_shared<vk_queue_handle_unsynchronized>(); + queue_info2.flags = eInternallySynchronizedKHR; + } else { + h = std::make_shared<vk_queue_handle_synchronized>(); } - if (!ggml_is_quantized(src0->type)) { - const size_t type_size = ggml_type_size(src0->type); - return type_size == 1 || type_size == 2 || type_size == 4 || type_size == 8; + h->queue = device->device.getQueue2(queue_info2); + h->device = device; + // Avoid concurrent submissions on NVIDIA due to driver bug. + if (device->vendor_id == VK_VENDOR_ID_NVIDIA) { + h->device_submit_mutex = &device->queue_submit_mutex; } + q->handle = h; - // Quantized tensor rows are block-aligned when created. - return ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(src1) && ggml_is_contiguous_rows(dst); -} + q->cmd_pool.init(device, q.get()); -template <typename T> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, T &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - GGML_UNUSED(p); - GGML_UNUSED(src0); - GGML_UNUSED(src1); - GGML_UNUSED(src2); - GGML_UNUSED(src3); - GGML_UNUSED(dst); - static_assert(!std::is_const<T>::value, "unexpected type"); - GGML_ASSERT(!src0 || get_misalign_bytes(ctx, src0) == 0); - GGML_ASSERT(!src1 || get_misalign_bytes(ctx, src1) == 0); - GGML_ASSERT(!src2 || get_misalign_bytes(ctx, src2) == 0); - GGML_ASSERT(!src3 || get_misalign_bytes(ctx, src3) == 0); - GGML_ASSERT(!dst || get_misalign_bytes(ctx, dst) == 0); + q->stage_flags = stage_flags; + return q; } -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_p021_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); - const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - - p.b_offset = b_offset; - p.d_offset = d_offset; - - GGML_UNUSED(src0); - GGML_UNUSED(src2); - GGML_UNUSED(src3); +std::unique_ptr<vk_queue> ggml_vk_create_aliased_queue(vk_device& device, const std::unique_ptr<vk_queue>& source) { + std::lock_guard<std::recursive_mutex> guard(device->mutex); + auto q = std::make_unique<vk_queue>(); + q->handle = source->handle; + q->queue_family_index = source->queue_family_index; + q->stage_flags = source->stage_flags; + q->transfer_only = source->transfer_only; + q->cmd_pool.init(device, q.get()); + return q; } -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_nc_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); - const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); +vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_command_pool& p) { + vk_context result = std::make_shared<vk_context_struct>(); + VK_LOG_DEBUG("ggml_vk_create_context(" << result << ")"); + ctx->gc.contexts.emplace_back(result); + result->p = &p; + return result; +} - p.b_offset = b_offset; - p.d_offset = d_offset; +vk_context ggml_vk_create_temporary_context(vk_command_pool& p) { + vk_context result = std::make_shared<vk_context_struct>(); + VK_LOG_DEBUG("ggml_vk_create_temporary_context(" << result << ")"); + result->p = &p; + return result; +} - GGML_UNUSED(src0); - GGML_UNUSED(src2); - GGML_UNUSED(src3); +static vk_semaphore * ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx) { + VK_LOG_DEBUG("ggml_vk_create_timeline_semaphore()"); + vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eBinary, 0 }; + vk::SemaphoreCreateInfo ci{}; + ci.setPNext(&tci); + vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci); + ctx->gc.semaphores.push_back({ semaphore, 0 }); + return &ctx->gc.semaphores[ctx->gc.semaphores.size() - 1]; } -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_fwht_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - p.src_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); - p.dst_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); +static vk_semaphore * ggml_vk_create_timeline_semaphore(ggml_backend_vk_context * ctx) { + VK_LOG_DEBUG("ggml_vk_create_timeline_semaphore()"); + if (ctx->semaphore_idx >= ctx->gc.tl_semaphores.size()) { + vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eTimeline, 0 }; + vk::SemaphoreCreateInfo ci{}; + ci.setPNext(&tci); + vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci); + ctx->gc.tl_semaphores.push_back({ semaphore, 0 }); + } + return &ctx->gc.tl_semaphores[ctx->semaphore_idx++]; +} - GGML_UNUSED(src1); - GGML_UNUSED(src2); - GGML_UNUSED(src3); +static vk::Event ggml_vk_create_event(ggml_backend_vk_context * ctx) { + if (ctx->event_idx >= ctx->gc.events.size()) { + ctx->gc.events.push_back(ctx->device->device.createEvent({})); + } + return ctx->gc.events[ctx->event_idx++]; } -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_turbo_wht_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - p.src_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); - p.dst_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - p.scale_offset = src1 ? get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type) : 0u; +void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p) { + VK_LOG_DEBUG("ggml_vk_command_pool_cleanup()"); - GGML_UNUSED(src2); - GGML_UNUSED(src3); + // Requires command buffers to be done + device->device.resetCommandPool(p.pool); + // Don't clear the command buffers and mark them as not in use. + // This allows us to reuse them + for (auto& cmd_buffer : p.cmd_buffers) { + cmd_buffer.in_use = false; + } } -struct ggml_backend_vk_buffer_context { - vk_device_ref device; - vk_buffer dev_buffer; - std::string name; +void ggml_vk_queue_command_pools_cleanup(vk_device& device) { + VK_LOG_DEBUG("ggml_vk_queue_command_pools_cleanup()"); - ggml_backend_vk_buffer_context(vk_device_ref device, vk_buffer&& dev_buffer, std::string& name) : - device(device), - dev_buffer(dev_buffer), - name(name) { - } + // Arbitrary frequency to cleanup/reuse command buffers + static constexpr uint32_t cleanup_frequency = 10; - ~ggml_backend_vk_buffer_context() { - ggml_vk_destroy_buffer(dev_buffer); + if (device->compute_queue && device->compute_queue->cmd_pool.buffers_in_use() >= cleanup_frequency) { + ggml_vk_command_pool_cleanup(device, device->compute_queue->cmd_pool); } -}; - -void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) { - if (!vk_memory_logger_enabled) { - return; + if (device->transfer_queue && device->transfer_queue->cmd_pool.buffers_in_use() >= cleanup_frequency) { + ggml_vk_command_pool_cleanup(device, device->transfer_queue->cmd_pool); } - std::lock_guard<std::mutex> guard(log_mutex); - vk_buffer buf = buf_ref.lock(); - const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal); - const std::string type = device ? "device" : "host"; - allocations[buf->buffer] = size; - total_device += device ? size : 0; - total_host += device ? 0 : size; - VK_LOG_MEMORY(buf->device->name << ": +" << format_size(size) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host)); } -void vk_memory_logger::log_deallocation(vk_buffer_ref buf_ref) { - if (buf_ref.expired() || buf_ref.lock()->size == 0 || !vk_memory_logger_enabled) { - return; - } - - std::lock_guard<std::mutex> guard(log_mutex); - vk_buffer buf = buf_ref.lock(); - const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal); - std::string type = device ? "device" : "host"; - auto it = allocations.find(buf->buffer); - if (it != allocations.end()) { - total_device -= device ? it->second : 0; - total_host -= device ? 0 : it->second; - VK_LOG_MEMORY(buf->device->name << ": -" << format_size(it->second) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host)); - allocations.erase(it); - } else { - VK_LOG_MEMORY("ERROR " << buf->device->name << ": Attempted to deallocate unknown " << type << " memory at " << buf->buffer); - } +vk_subbuffer ggml_vk_subbuffer(const ggml_backend_vk_context* ctx, const vk_buffer& buf, size_t offset) { + return { buf, offset, ggml_vk_get_max_buffer_range(ctx, buf, offset) }; } -struct vk_instance_t { - vk::Instance instance; +void ggml_vk_sync_buffers(ggml_backend_vk_context* ctx, vk_context& subctx) { + VK_LOG_DEBUG("ggml_vk_sync_buffers()"); - bool debug_utils_support = false; // VK_EXT_debug_utils enabled - PFN_vkSetDebugUtilsObjectNameEXT pfn_vkSetDebugUtilsObjectNameEXT = {}; - PFN_vkQueueBeginDebugUtilsLabelEXT pfn_vkQueueBeginDebugUtilsLabelEXT = {}; - PFN_vkQueueEndDebugUtilsLabelEXT pfn_vkQueueEndDebugUtilsLabelEXT = {}; - PFN_vkCmdBeginDebugUtilsLabelEXT pfn_vkCmdBeginDebugUtilsLabelEXT = {}; - PFN_vkCmdEndDebugUtilsLabelEXT pfn_vkCmdEndDebugUtilsLabelEXT = {}; - PFN_vkCmdInsertDebugUtilsLabelEXT pfn_vkCmdInsertDebugUtilsLabelEXT = {}; + ggml_vk_flush_cpy_regions(subctx); - std::vector<size_t> device_indices; - std::vector<bool> device_supports_membudget; - vk_device devices[GGML_VK_MAX_DEVICES]; -}; + const bool transfer_queue = subctx->p->q->transfer_only; -static bool vk_instance_initialized = false; -static vk_instance_t vk_instance; + if (ctx) { + ctx->prealloc_x_need_sync = ctx->prealloc_y_need_sync = ctx->prealloc_split_k_need_sync = false; + } -#ifdef GGML_VULKAN_CHECK_RESULTS -static size_t vk_skip_checks; -static size_t vk_output_tensor; + subctx->s->buffer->buf.pipelineBarrier( + subctx->p->q->stage_flags, + subctx->p->q->stage_flags, + {}, + { { + { !transfer_queue ? (vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite) : (vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite) }, + { !transfer_queue ? (vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite) : (vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite) } + } }, + {}, + {} + ); +} -static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name); -static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx); -static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx); -#endif +static void ggml_vk_reset_event(vk_context& ctx, vk::Event& event) { + VK_LOG_DEBUG("ggml_vk_set_event()"); -typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + ctx->s->buffer->buf.resetEvent( + event, + ctx->p->q->stage_flags + ); +} -static void ggml_backend_vk_free(ggml_backend_t backend); +void ggml_vk_set_event(vk_context& ctx, vk::Event& event) { + VK_LOG_DEBUG("ggml_vk_set_event()"); -static VkDeviceSize ggml_vk_get_max_buffer_range(const ggml_backend_vk_context * ctx, const vk_buffer &buf, const VkDeviceSize offset) { - const VkDeviceSize range = std::min(VkDeviceSize{buf->size - offset}, - VkDeviceSize{ctx->device->properties.limits.maxStorageBufferRange}); - return range; + ctx->s->buffer->buf.setEvent( + event, + ctx->p->q->stage_flags + ); } -// Wait for ctx->fence to be signaled. -static void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) { - // Use waitForFences while most of the graph executes. Hopefully the CPU can sleep - // during this wait. - if (ctx->almost_ready_fence_pending) { - VK_CHECK(ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX), "almost_ready_fence", ctx->device); - ctx->device->device.resetFences({ ctx->almost_ready_fence }); - ctx->almost_ready_fence_pending = false; +void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events) { + VK_LOG_DEBUG("ggml_vk_wait_events()"); + if (events.empty()) { + return; } - // Spin (w/pause) waiting for the graph to finish executing. - vk::Result result; - for (;;) { - try { - result = ctx->device->device.getFenceStatus(ctx->fence); - } catch (vk::DeviceLostError &) { - ggml_vk_print_device_lost_info(ctx->device); - GGML_LOG_ERROR("ggml_vulkan: getFenceStatus at %s:%d\n", __FILE__, __LINE__); - throw; - } - if (result == vk::Result::eSuccess) { - break; - } - if (result != vk::Result::eNotReady) { - GGML_LOG_ERROR("ggml_vulkan: error %s at %s:%d\n", to_string(result).c_str(), __FILE__, __LINE__); - throw vk::SystemError(vk::make_error_code(result), "ggml_vulkan: getFenceStatus"); - } - for (uint32_t i = 0; i < 100; ++i) { - YIELD(); - YIELD(); - YIELD(); - YIELD(); - YIELD(); - YIELD(); - YIELD(); - YIELD(); - YIELD(); - YIELD(); - } - } - ctx->device->device.resetFences({ ctx->fence }); + ctx->s->buffer->buf.waitEvents( + events, + ctx->p->q->stage_flags, + ctx->p->q->stage_flags, + {}, + {}, + {} + ); } -static constexpr uint32_t kSpvOpCooperativeMatrixLoadTensorNV = 5367; -static constexpr uint32_t kSpvCapabilityCooperativeMatrixDecodeVectorNV = 5447; -static constexpr uint32_t kSpvTensorAddressingDecodeVectorFuncBit = 0x4; +static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) { -// Remove SPV_NV_cooperative_matrix_decode_vector usage from a SPIR-V module so it -// can be loaded on drivers that only support SPV_NV_cooperative_matrix2. Drops the -// OpExtension declaration, the CooperativeMatrixDecodeVectorNV OpCapability, and the -// DecodeVectorFunc operand from any OpCooperativeMatrixLoadTensorNV instruction. -// Returns true when the input used the extension (and `out` was populated with a -// stripped copy); returns false otherwise without touching `out`. -static bool ggml_vk_strip_decode_vector(const uint32_t * code, size_t word_count, std::vector<uint32_t> & out) { - static const char kDecodeVectorExt[] = "SPV_NV_cooperative_matrix_decode_vector"; + vk_fa_tuning_params result{}; + result.path = FA_SCALAR; - if (word_count < 5) { - return false; + if (device->vendor_id == VK_VENDOR_ID_INTEL) { + // Disable subgroup use due to performance issues when enforcing subgroup sizes + result.subgroup_size = 32; + result.disable_subgroups = true; + } else if (device->vendor_id == VK_VENDOR_ID_AMD && device->architecture != AMD_GCN) { + result.subgroup_size = n_rows < 4 ? 32 : device->subgroup_size; + } else { + result.subgroup_size = device->subgroup_size; } - bool uses_decode_vector = false; - for (size_t pos = 5; pos < word_count; ) { - uint32_t word = code[pos]; - uint32_t wc = word >> spv::WordCountShift; - uint32_t op = word & spv::OpCodeMask; - GGML_ASSERT(wc > 0 && pos + wc <= word_count); - if (op == spv::OpExtension && wc >= 2) { - const char * s = reinterpret_cast<const char *>(&code[pos + 1]); - if (strcmp(s, kDecodeVectorExt) == 0) { - uses_decode_vector = true; - break; - } - } - pos += wc; + // Row split splits the workgroup so that synchronization only has to happen within subgroups, which avoids barriers + uint32_t row_split_max_hsk = 64; + if (device->vendor_id == VK_VENDOR_ID_AMD && device->architecture != AMD_GCN && !device->uma) { + row_split_max_hsk = n_rows <= 8 ? 64 : 128; } + result.row_split = (n_rows < 4 || hsk <= row_split_max_hsk) ? 1 : 4; - if (!uses_decode_vector) { - return false; + if (result.subgroup_size > 32 && (n_rows < 4 || hsk < (result.row_split == 1 ? 128 : 64))) { + result.workgroup_size = result.subgroup_size * 2; + } else { + result.workgroup_size = result.subgroup_size * 4; } - VK_LOG_DEBUG("ggml_vk_strip_decode_vector: stripping SPV_NV_cooperative_matrix_decode_vector"); + const uint32_t D = hsk | hsv; - // Bulk-copy unchanged runs and only break the run when an instruction needs to - // be dropped or patched. Use reserve + insert/push_back so the destination buffer - // is touched exactly once (no zero-initialization pass from resize()). - out.clear(); - out.reserve(word_count); + const bool reduce_block_rows = D & 8 || n_kv < 1024 || device->vendor_id == VK_VENDOR_ID_INTEL; - size_t run_start = 0; - auto flush_run = [&](size_t up_to) { - if (up_to > run_start) { - out.insert(out.end(), code + run_start, code + up_to); + if (n_rows == 1) { + result.block_rows = 1; + result.block_cols = 64; + } else { + // row_split 1 means higher register use per row, so block size has to be adjusted + if (result.row_split == 1) { + result.block_rows = n_rows == 2 ? 2 : ((n_rows <= 4 || reduce_block_rows) ? 4 : 8); + } else { + result.block_rows = n_rows <= 4 ? 4 : ((n_rows <= 8 || reduce_block_rows) ? 8 : 16); } - }; - for (size_t pos = 5; pos < word_count; ) { - uint32_t word = code[pos]; - uint32_t wc = word >> spv::WordCountShift; - uint32_t op = word & spv::OpCodeMask; - GGML_ASSERT(wc > 0 && pos + wc <= word_count); + result.block_cols = (D & 8) ? 64 : 32; + } - if (op == spv::OpExtension && wc >= 2) { - const char * s = reinterpret_cast<const char *>(&code[pos + 1]); - if (strcmp(s, kDecodeVectorExt) == 0) { - flush_run(pos); - pos += wc; - run_start = pos; - continue; - } - } + const uint32_t D_lsb = D ^ (D & (D-1)); // extract lowest set bit - if (op == spv::OpCapability && wc == 2 && code[pos + 1] == kSpvCapabilityCooperativeMatrixDecodeVectorNV) { - flush_run(pos); - pos += wc; - run_start = pos; - continue; - } + result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4); - if (op == kSpvOpCooperativeMatrixLoadTensorNV) { - // [opcode/wc][ResultType][Result][Pointer][Object][TensorLayout][MemOperand mask][mem extras...][TA mask][ta extras...] - GGML_ASSERT(wc >= 8); + result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0; - uint32_t mem_mask = code[pos + 6]; - size_t cur = pos + 7; - // Each of these MemoryAccess bits (when set) carries one trailing operand. - cur += (mem_mask & 0x2) ? 1 : 0; // Aligned - cur += (mem_mask & 0x8) ? 1 : 0; // MakePointerAvailable - cur += (mem_mask & 0x10) ? 1 : 0; // MakePointerVisible - cur += (mem_mask & 0x10000) ? 1 : 0; // AliasScopeINTELMask - cur += (mem_mask & 0x20000) ? 1 : 0; // NoAliasINTELMask - GGML_ASSERT(cur < pos + wc); + if (!reduce_block_rows && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) { + result.block_rows /= 2; + } - uint32_t ta_mask = code[cur]; - if ((ta_mask & kSpvTensorAddressingDecodeVectorFuncBit) == 0) { - pos += wc; - continue; // leave instruction inside the current unchanged run - } + // On AMD RDNA, for small head sizes and big batch size the shader uses few registers, so too many subgroups get scheduled + // at once and end up thrashing the cache. Fix this by setting a large (unused) shmem buffer that reduces occupancy. + // This targets an occupancy of 4 subgroups per SIMD. + if (device->vendor_id == VK_VENDOR_ID_AMD && device->properties.limits.maxComputeSharedMemorySize == 65536) { + if (device->architecture != AMD_GCN && n_rows >= 64 && hsk <= 128) { + // 30kb target for hsk > 64, 26kb for <= 64 due to smaller workgroup size + // Values are guessed, tested on RDNA2 + result.limit_occupancy_shmem = (hsk <= 64 ? 26 : 30) * 1024 / 4 / 4; + } else if (device->architecture == AMD_GCN && n_rows <= 8 && hsk >= 256) { + // Same thing for GCN, with an occupancy target of 2 subgroups per SIMD. + // Here low-batch FA with large head size is affected. + // n_rows < 4 switch because workgroup size switches from 128 to 256 there. + result.limit_occupancy_shmem = (n_rows < 4 ? 14 : 26) * 1024 / 4 / 4; + } + } - flush_run(pos); + return result; +} - // Append unchanged prefix of the instruction (header through the mem-extras). - size_t inst_start = out.size(); - size_t pre_n = cur - pos; - out.insert(out.end(), code + pos, code + pos + pre_n); +static vk_fa_tuning_params get_fa_tuning_params_coopmat1(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) { + GGML_UNUSED(n_rows); + GGML_UNUSED(n_kv); + GGML_UNUSED(k_type); + GGML_UNUSED(v_type); + GGML_UNUSED(f32acc); - // Emit TA mask with the DecodeVectorFunc bit cleared. - out.push_back(ta_mask & ~kSpvTensorAddressingDecodeVectorFuncBit); + vk_fa_tuning_params result{}; + result.path = FA_COOPMAT1; - // TA extras: TensorView (0x1) and DecodeFunc (0x2) are kept verbatim; - // DecodeVectorFunc (0x4) is dropped along with its trailing id operand. - size_t keep_ta_extras = ((ta_mask & 0x1) ? 1 : 0) + ((ta_mask & 0x2) ? 1 : 0); - if (keep_ta_extras) { - out.insert(out.end(), code + cur + 1, code + cur + 1 + keep_ta_extras); - } + const uint32_t D = hsk | hsv; - GGML_ASSERT(wc == pre_n + 1 + keep_ta_extras + 1); + const uint32_t coopmat_block_rows = 16; + const uint32_t coopmat_block_cols = 16; - // Patch the instruction header with the new (one-shorter) word count. - uint32_t new_wc = wc - 1; - out[inst_start] = (new_wc << spv::WordCountShift) | op; + const uint32_t num_subgroups = 4; - pos += wc; - run_start = pos; - continue; - } + result.block_rows = coopmat_block_rows; + result.block_cols = coopmat_block_cols * num_subgroups; + result.row_split = num_subgroups; + result.subgroup_size = device->subgroup_size; + result.workgroup_size = num_subgroups * result.subgroup_size; - pos += wc; - } + const uint32_t D_lsb = D ^ (D & (D-1)); // extract lowest set bit + result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4); - flush_run(word_count); - return true; + result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0; + + return result; } -// Remove the loop unrolling hint of the matmul shader's BK loop -// and replace it with the dont_unroll hint for better performance on -// hardware like Apple M1/M2. -// Assumes 1. code comes from mul_mm.comp 2. the K-tile loop has no loop -// control hint and 3. the BK loop is the last loop nested directly inside -// the K-tile loop. -// Returns true when the input was modified; returns false otherwise -// without touching `out`. -static bool ggml_vk_roll_bk_loop(const uint32_t * code, size_t word_count, std::vector<uint32_t> & out) { - if (word_count < 5) { - return false; - } +static vk_fa_tuning_params get_fa_tuning_params_coopmat2(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) { + GGML_UNUSED(n_kv); + GGML_UNUSED(f32acc); - struct vk_spv_loop { - size_t header; - size_t end; - uint32_t control; - }; + vk_fa_tuning_params result{}; + result.path = FA_COOPMAT2; - std::vector<vk_spv_loop> loops; + const uint32_t D = hsk | hsv; - // Collect a list of all loops in the module. - for (size_t pos = 5; pos < word_count; ) { - const uint32_t wc = code[pos] >> spv::WordCountShift; - const uint32_t op = code[pos] & spv::OpCodeMask; - if (wc == 0 || pos + wc > word_count) { - return false; - } + const bool small_rows = n_rows < 32; - if (op == spv::OpLoopMerge && wc >= 4) { loops.push_back({ pos, 0, code[pos + 3] }); } + if (small_rows) { + result.block_rows = 32; + result.block_cols = 32; + } else if (ggml_is_quantized(k_type) || ggml_is_quantized(v_type) || hsk >= 256 || hsv >= 256) { + result.block_rows = (hsk >= 512 || hsv >= 512) ? 32 : 64; + result.block_cols = 32; + } else { + result.block_rows = 64; + result.block_cols = 64; + } - if (op == spv::OpLabel && wc >= 2) { - for (auto & l : loops) { - if (l.end == 0 && code[l.header + 1] == code[pos + 1]) { l.end = pos; } - } - } + result.subgroup_size = device->subgroup_size; + result.workgroup_size = (small_rows && (D % 32) == 0) ? 256 : 128; - pos += wc; + return result; +} + +vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) { + FaCodePath path = device->coopmat2 ? FA_COOPMAT2 : + device->coopmat1_fa_support ? FA_COOPMAT1 : FA_SCALAR; + + if (path == FA_COOPMAT2 && k_type == GGML_TYPE_BF16 && !device->coopmat2_bf16_support) { + path = FA_COOPMAT1; + } + if (path == FA_COOPMAT1 && k_type == GGML_TYPE_BF16 && !device->coopmat_bf16_support) { + path = FA_SCALAR; } - auto encloses = [](const vk_spv_loop & a, const vk_spv_loop & b) { - return a.header < b.header && b.header < a.end; - }; + if (path == FA_COOPMAT1 && device->architecture == vk_device_architecture::NVIDIA_TURING) { + // Nvidia compiler bug, see https://github.com/ggml-org/llama.cpp/pull/19075#issuecomment-3820716090 + path = FA_SCALAR; + } - // Find the BK loop. - const vk_spv_loop * bk = nullptr; - for (const auto & h : loops) { - if (h.control != spv::LoopControlUnrollMask) { - continue; - } - const vk_spv_loop * parent = nullptr; - bool has_child = false; - for (const auto & g : loops) { - if (encloses(g, h) && (!parent || g.header > parent->header)) { - parent = &g; - } - if (encloses(h, g)) { - has_child = true; - } - } - // BK loop should be the last loop nested inside the loop with no hint - // and have at least one child loop. - if (parent && - parent->control == spv::LoopControlMaskNone && - has_child && - (!bk || h.header > bk->header)) { - bk = &h; + if (path == FA_COOPMAT1) { + bool shape_ok = (f32acc && device->coopmat_support_16x16x16_f32acc) || + (!f32acc && device->coopmat_support_16x16x16_f16acc); + const vk_fa_tuning_params params = get_fa_tuning_params_coopmat1(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc); + bool shmem_ok = ggml_vk_flash_attn_coopmat_shmem_support(device, params, hsk, hsv, f32acc, k_type, v_type); + + if (!shape_ok || !shmem_ok) { + path = FA_SCALAR; } } - if (!bk) { - return false; + + // scalar is faster than coopmat when N==1 + if (n_rows == 1 && (path == FA_COOPMAT1 || path == FA_COOPMAT2)) { + path = FA_SCALAR; } - // set DontUnroll instead of Unroll - out.assign(code, code + word_count); - out[bk->header + 3] = spv::LoopControlDontUnrollMask; - return true; + switch (path) { + case FA_SCALAR: + return get_fa_tuning_params_scalar(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc); + case FA_COOPMAT1: + return get_fa_tuning_params_coopmat1(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc); + case FA_COOPMAT2: + return get_fa_tuning_params_coopmat2(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc); + default: + throw std::runtime_error("unsupported FaCodePath"); + } } -static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipeline, size_t spv_size, const void* spv_data, const std::string entrypoint, - uint32_t parameter_count, std::array<uint32_t, 3> wg_denoms, std::vector<uint32_t> specialization_constants, - bool disable_robustness, bool require_full_subgroups, uint32_t required_subgroup_size) { - VK_LOG_DEBUG("ggml_vk_create_pipeline(" << device->name << ", " << pipeline->name << ", " << entrypoint << ", " << parameter_count << - ", (" << wg_denoms[0] << "," << wg_denoms[1] << "," << wg_denoms[2] << "), specialization_constants, " << - disable_robustness << ", " << require_full_subgroups << ", " << required_subgroup_size << ")"); - GGML_ASSERT(parameter_count > 0); - GGML_ASSERT(parameter_count <= MAX_PARAMETER_COUNT); - GGML_ASSERT(wg_denoms[0] > 0 && wg_denoms[1] > 0 && wg_denoms[2] > 0); // NOLINT - - vk::ShaderModuleCreateInfo shader_module_create_info({}, spv_size, reinterpret_cast<const uint32_t *>(spv_data)); +vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc, + bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, ggml_type k_type, ggml_type v_type) { + const bool old_amd_windows = device->vendor_id == VK_VENDOR_ID_AMD && device->driver_id == vk::DriverId::eAmdProprietary && + (device->architecture == AMD_GCN || device->architecture == AMD_RDNA1 || device->architecture == AMD_RDNA2); - // Patch SPIR-V to enable supported FP16 float controls, avoiding the need - // for separate shader variants. - std::vector<uint32_t> spirv; - if (device->float_controls_rte_fp16 || device->float_controls_denorm_preserve_fp16) { - const uint32_t* spv_words = reinterpret_cast<const uint32_t *>(spv_data); - size_t word_count = spv_size / sizeof(uint32_t); - spirv.assign(spv_words, spv_words + word_count); + uint32_t flags = (use_mask_opt ? 1 : 0) | + (use_mask ? 2 : 0) | + (use_logit_softcap ? 4 : 0) | + (old_amd_windows ? 8 : 0) | + (use_sparse ? 16 : 0); - // Find insertion points respecting SPIR-V layout order: - // Header(5) -> OpCapability -> OpExtension -> ... -> OpEntryPoint -> OpExecutionMode -> ... - size_t pos = 5; // skip header - size_t cap_insert_pos = pos; - size_t ext_insert_pos = pos; - size_t exec_insert_pos = pos; - uint32_t entry_point_id = 0; + const uint32_t subgroup_size = params.disable_subgroups ? 0 : params.subgroup_size; - while (pos < spirv.size()) { - uint32_t opcode = spirv[pos] & spv::OpCodeMask; - uint32_t len = spirv[pos] >> spv::WordCountShift; - if (len == 0) break; + return vk_fa_pipeline_state{hsk, hsv, params.block_rows, params.block_cols, params.d_split, params.row_split, params.shmem_staging, params.path, params.workgroup_size, subgroup_size, aligned, f32acc, flags, params.limit_occupancy_shmem, k_type, v_type}; +} - if (opcode == spv::OpCapability) { - cap_insert_pos = pos + len; - ext_insert_pos = pos + len; - } else if (opcode == spv::OpExtension) { - ext_insert_pos = pos + len; - } else if (opcode == spv::OpEntryPoint) { - entry_point_id = spirv[pos + 2]; - exec_insert_pos = pos + len; - } else if (opcode == spv::OpExecutionMode || opcode == spv::OpExecutionModeId) { - exec_insert_pos = pos + len; - } else if (entry_point_id != 0) { - break; - } - - pos += len; - } +static uint32_t fa_block_bytes(ggml_type t) { + if (t == GGML_TYPE_F32) { + return 16u; + } + return (uint32_t) ggml_type_size(t); +} - // Insert from latest position first so earlier indices stay valid. +static std::vector<uint32_t> get_fa_spec_constants(const vk_fa_pipeline_state& state) { + return { + /* 0 WorkGroupSize */ state.workgroup_size, + /* 1 Br */ state.Br, + /* 2 Bc */ state.Bc, + /* 3 HSK */ state.HSK, + /* 4 HSV */ state.HSV, + /* 5 Clamp */ static_cast<uint32_t>(!state.aligned), + /* 6 D_split */ state.D_split, + /* 7 row_split */ state.row_split, + /* 8 SubGroupSize */ state.subgroup_size, + /* 9 SHMEM_STAGING */ state.shmem_staging ? 1u : 0u, + /*10 Flags */ state.flags, + /*11 LIMIT_OCCUPANCY_SHMEM */ state.limit_occupancy_shmem, + /*12 FaTypeK */ static_cast<uint32_t>(state.k_type), + /*13 FaTypeV */ static_cast<uint32_t>(state.v_type), + /*14 FaBlockBytesK */ fa_block_bytes(state.k_type), + /*15 FaBlockBytesV */ fa_block_bytes(state.v_type), + }; +} - if (device->float_controls_rte_fp16) { - // OpExecutionMode %entrypoint RoundingModeRTE 16 - uint32_t exec_mode[] = { (4u << spv::WordCountShift) | spv::OpExecutionMode, entry_point_id, spv::ExecutionModeRoundingModeRTE, 16 }; - spirv.insert(spirv.begin() + exec_insert_pos, std::begin(exec_mode), std::end(exec_mode)); - } +static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) { - if (device->float_controls_denorm_preserve_fp16) { - // OpExecutionMode %entrypoint DenormPreserve 16 - uint32_t exec_mode[] = { (4u << spv::WordCountShift) | spv::OpExecutionMode, entry_point_id, spv::ExecutionModeDenormPreserve, 16 }; - spirv.insert(spirv.begin() + exec_insert_pos, std::begin(exec_mode), std::end(exec_mode)); - } + uint32_t lut_size = 0; + switch (src0_type) { + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + // Regular matmul uses the compact uint16_t IQ1 grid; the expanded + // uint32_t grid is only enabled for the q8_1/int-dot vector path. + lut_size = 2*2048; + break; + case GGML_TYPE_IQ2_XXS: + lut_size = 8*256; + break; + case GGML_TYPE_IQ2_XS: + lut_size = 8*512; + break; + case GGML_TYPE_IQ2_S: + lut_size = 8*1024; + break; + case GGML_TYPE_IQ3_XXS: + lut_size = 4*256; + break; + case GGML_TYPE_IQ3_S: + lut_size = 4*512; + break; + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_MXFP4: + lut_size = 4*16; + break; + case GGML_TYPE_NVFP4: + // Same kvalues budget as MXFP4 plus ue4m3_fp32_lut[128] (types.glsl, DATA_A_NVFP4). + lut_size = 4*16 + 128u * (uint32_t)sizeof(float); + break; + default: + break; + } - // OpExtension "SPV_KHR_float_controls" - const char ext_str[] = "SPV_KHR_float_controls"; - size_t ext_str_words = CEIL_DIV(sizeof(ext_str), sizeof(uint32_t)); - std::vector<uint32_t> extension(1 + ext_str_words, 0); - extension[0] = (uint32_t)((1 + ext_str_words) << spv::WordCountShift) | spv::OpExtension; - memcpy(&extension[1], ext_str, sizeof(ext_str)); - spirv.insert(spirv.begin() + ext_insert_pos, extension.begin(), extension.end()); + // Needs to be kept up to date on shader changes + // Needs to stay aligned with ggml_vk_mul_mm_spec. + const bool intel_shmem_stride_pad_zero = device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && + device->driver_id == vk::DriverId::eIntelProprietaryWindows; + const uint32_t bank_conflict_offset = intel_shmem_stride_pad_zero ? 0 : (device->coopmat_support ? 8 : 1); + const uint32_t type_size = device->fp16 ? sizeof(ggml_fp16_t) : sizeof(float); + const uint32_t warps = warptile[0] / warptile[10]; - if (device->float_controls_rte_fp16) { - // OpCapability RoundingModeRTE - uint32_t capability[] = { (2u << spv::WordCountShift) | spv::OpCapability, spv::CapabilityRoundingModeRTE }; - spirv.insert(spirv.begin() + cap_insert_pos, std::begin(capability), std::end(capability)); - } + const uint32_t load_bufs = (warptile[1] + warptile[2]) * (warptile[3] + bank_conflict_offset) * type_size; + const uint32_t mmid_row_ids = mul_mat_id ? (warptile[2] * 2 * sizeof(uint16_t)) : 0; + const uint32_t coopmat_stage = device->coopmat_support ? warptile[7] * warptile[8] / warps * sizeof(float) : 0; + const uint32_t ballots_sh = mul_mat_id ? (warps * 4 * sizeof(uint32_t)) : 0; - if (device->float_controls_denorm_preserve_fp16) { - // OpCapability DenormPreserve - uint32_t capability[] = { (2u << spv::WordCountShift) | spv::OpCapability, spv::CapabilityDenormPreserve }; - spirv.insert(spirv.begin() + cap_insert_pos, std::begin(capability), std::end(capability)); - } + const uint32_t total_size = load_bufs + mmid_row_ids + coopmat_stage + lut_size + ballots_sh; + const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; - shader_module_create_info = vk::ShaderModuleCreateInfo({}, spirv.size() * sizeof(uint32_t), spirv.data()); - } + VK_LOG_DEBUG("ggml_vk_matmul_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), " + "mul_mat_id=" << mul_mat_id << ", src0_type=" << ggml_type_name(src0_type) << ", supported=" << supported); -#if defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT) - if (device->coopmat2 && !device->coopmat2_decode_vector) { - const uint32_t * src = spirv.empty() ? reinterpret_cast<const uint32_t *>(spv_data) : spirv.data(); - size_t src_n = spirv.empty() ? spv_size / sizeof(uint32_t) : spirv.size(); - std::vector<uint32_t> stripped; - if (ggml_vk_strip_decode_vector(src, src_n, stripped)) { - spirv = std::move(stripped); - shader_module_create_info = vk::ShaderModuleCreateInfo({}, spirv.size() * sizeof(uint32_t), spirv.data()); - } - } -#endif + return supported; +} -#if VK_HEADER_VERSION >= 287 - // Roll the mul_mm BK loop on Asahi Linux. Skip bf16 and the mul_mmq pipelines. - if (device->driver_id == vk::DriverId::eMesaHoneykrisp && - pipeline->name.rfind("matmul", 0) == 0 && - pipeline->name.find("bf16") == std::string::npos && - pipeline->name.find("q8_1") == std::string::npos) { - const uint32_t * src = spirv.empty() ? reinterpret_cast<const uint32_t *>(spv_data) : spirv.data(); - size_t src_n = spirv.empty() ? spv_size / sizeof(uint32_t) : spirv.size(); - std::vector<uint32_t> rolled; - if (ggml_vk_roll_bk_loop(src, src_n, rolled)) { - spirv = std::move(rolled); - shader_module_create_info = vk::ShaderModuleCreateInfo({}, spirv.size() * sizeof(uint32_t), spirv.data()); - } - } -#endif +static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) { - pipeline->shader_module = device->device.createShaderModule(shader_module_create_info); + // FLOAT_TYPE in the shader is float16_t with fp16 support, otherwise float. + const uint32_t fp_size = device->fp16 ? 2u : 4u; + const uint32_t fp_align = fp_size; + const uint32_t fp2_size = 2u * fp_size; + const uint32_t fp2_align = device->fp16 ? 4u : 8u; - vk::PushConstantRange pcr( - vk::ShaderStageFlagBits::eCompute, - 0, - pipeline->push_constant_size - ); + struct member { uint32_t size, align; }; + auto std430_size = [](std::initializer_list<member> members) { + uint32_t off = 0, struct_align = 1; + for (const auto &m : members) { + off = (off + m.align - 1) & ~(m.align - 1); + off += m.size; + struct_align = std::max(struct_align, m.align); + } + return (off + struct_align - 1) & ~(struct_align - 1); + }; - vk::PipelineLayoutCreateInfo pipeline_layout_create_info(vk::PipelineLayoutCreateFlags(), device->dsl, pcr); - pipeline->layout = device->device.createPipelineLayout(pipeline_layout_create_info); + uint32_t block_a_size = 0; + switch (src0_type) { + case GGML_TYPE_Q2_0: block_a_size = std430_size({{32, 4}, {fp_size, fp_align}}); break; // qs[8] + dm + case GGML_TYPE_Q4_0: block_a_size = std430_size({{16, 4}, {fp_size, fp_align}}); break; // qs[16/4] + dm + case GGML_TYPE_Q4_1: block_a_size = std430_size({{16, 4}, {fp2_size, fp2_align}}); break; // qs[16/4] + dm(vec2) + case GGML_TYPE_Q5_0: block_a_size = std430_size({{16, 4}, {4, 4}, {fp_size, fp_align}}); break; // qs[16/4] + qh + dm + case GGML_TYPE_Q5_1: block_a_size = std430_size({{16, 4}, {4, 4}, {fp2_size, fp2_align}}); break; // qs[16/4] + qh + dm(vec2) + case GGML_TYPE_Q8_0: block_a_size = std430_size({{32, 4}, {fp_size, fp_align}}); break; // qs[8] + dm + case GGML_TYPE_IQ4_XS: block_a_size = std430_size({{32, 4}, {fp_size, fp_align}}); break; // qs[8] + d + case GGML_TYPE_MXFP4: block_a_size = std430_size({{32, 4}, {fp_size, fp_align}}); break; // qs[8] + d + case GGML_TYPE_Q2_K: block_a_size = std430_size({{ 8, 4}, {2, 2}, {fp2_size, fp2_align}}); break; // qs[2] + scales(u8vec2) + dm(vec2) + case GGML_TYPE_Q3_K: block_a_size = std430_size({{16, 4}, {fp2_size, fp2_align}}); break; // qs[4] + d_scales(vec2) + case GGML_TYPE_Q4_K: block_a_size = std430_size({{16, 4}, {fp2_size, fp2_align}}); break; // qs[4] + dm(vec2) + case GGML_TYPE_Q5_K: block_a_size = std430_size({{32, 4}, {fp2_size, fp2_align}}); break; // qs[8] + dm(vec2) + case GGML_TYPE_Q6_K: block_a_size = std430_size({{32, 4}, {fp2_size, fp2_align}}); break; // qs[8] + d_scales(vec2) + case GGML_TYPE_IQ3_S: block_a_size = std430_size({{32, 4}, {fp_size, fp_align}}); break; // qs[8] + d + default: + return false; + } - std::vector<vk::SpecializationMapEntry> specialization_entries(specialization_constants.size()); + // IQ3_S also copies its 512-entry grid into shared memory (types.glsl, init_iq_shmem) + const uint32_t lut_size = (src0_type == GGML_TYPE_IQ3_S) ? 4*512 : 0; - for (size_t i = 0; i < specialization_constants.size(); i++) { - specialization_entries[i].constantID = i; - specialization_entries[i].offset = i * sizeof(uint32_t); - specialization_entries[i].size = sizeof(uint32_t); - } + // block_b_cache: { int32_t qs[8]; FLOAT_TYPEV2 ds; } + const uint32_t block_b_size = std430_size({{32, 4}, {fp2_size, fp2_align}}); - vk::SpecializationInfo specialization_info( - specialization_entries.size(), - specialization_entries.data(), - specialization_constants.size() * sizeof(uint32_t), - specialization_constants.data() - ); + const uint32_t BM = warptile[1]; + const uint32_t BN = warptile[2]; + // mul_mmq.comp: BK_STEP=1 for MUL_MAT_ID, 4 otherwise. + const uint32_t BK_STEP = mul_mat_id ? 1u : 4u; - vk::PipelineShaderStageCreateFlags pipeline_shader_stage_create_flags{}; + const uint32_t buf_a_size = BM * BK_STEP * block_a_size; + const uint32_t buf_b_size = BN * BK_STEP * block_b_size; + const uint32_t mmid_row_ids = mul_mat_id ? (BN * 2u * (uint32_t)sizeof(uint16_t)) : 0u; - if (device->subgroup_require_full_support && require_full_subgroups) { - pipeline_shader_stage_create_flags |= vk::PipelineShaderStageCreateFlagBits::eRequireFullSubgroupsEXT; - } + const uint32_t warps = warptile[0] / warptile[10]; + const uint32_t ballots_sh = mul_mat_id ? (warps * 4u * (uint32_t)sizeof(uint32_t)) : 0u; - vk::PipelineShaderStageCreateInfo pipeline_shader_create_info( - pipeline_shader_stage_create_flags, - vk::ShaderStageFlagBits::eCompute, - pipeline->shader_module, - entrypoint.c_str(), - &specialization_info); + const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + ballots_sh + lut_size; + const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; - vk::PipelineShaderStageRequiredSubgroupSizeCreateInfoEXT pipeline_shader_stage_required_subgroup_size_create_info; - pipeline_shader_stage_required_subgroup_size_create_info.requiredSubgroupSize = required_subgroup_size; - if (device->subgroup_size_control && required_subgroup_size > 0) { - GGML_ASSERT(device->subgroup_min_size <= required_subgroup_size && required_subgroup_size <= device->subgroup_max_size); - pipeline_shader_create_info.setPNext(&pipeline_shader_stage_required_subgroup_size_create_info); - } + VK_LOG_DEBUG("ggml_vk_matmul_int_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), " + "mul_mat_id=" << mul_mat_id << ", src0_type=" << ggml_type_name(src0_type) << ", total=" << total_size << ", supported=" << supported); - vk::ComputePipelineCreateInfo compute_pipeline_create_info( - device->pipeline_executable_properties_support ? - vk::PipelineCreateFlagBits::eCaptureStatisticsKHR : - vk::PipelineCreateFlags{}, - pipeline_shader_create_info, - pipeline->layout); + return supported; +} - vk::PipelineRobustnessCreateInfoEXT rci; +static const std::unordered_map<std::string, uint32_t> rdna1_pipelines = { + {"soft_max", 64}, {"im2col", 64}, + {"argmax", 64}, {"mul_mat_vec", 64}, + {"mul_mat_vec_f16", 32}, {"mul_mat_vec_f32_f16", 32} +}; - if (device->pipeline_robustness && disable_robustness) { - rci.storageBuffers = vk::PipelineRobustnessBufferBehaviorEXT::eDisabled; - rci.uniformBuffers = vk::PipelineRobustnessBufferBehaviorEXT::eDisabled; - compute_pipeline_create_info.setPNext(&rci); - } +static const std::unordered_map<std::string, uint32_t> rdna2_pipelines = { + {"soft_max", 64}, {"im2col", 64}, +}; -#if defined(VK_EXT_shader_64bit_indexing) - vk::PipelineCreateFlags2CreateInfo pipelineFlags2CreateInfo; - if (pipeline->is_64b_indexing) +static std::vector<GpuPipelineConfig> gpu_pipeline_configs = { { - pipelineFlags2CreateInfo.flags = vk::PipelineCreateFlagBits2::e64BitIndexingEXT; - if (device->pipeline_executable_properties_support) { - pipelineFlags2CreateInfo.flags |= vk::PipelineCreateFlagBits2::eCaptureStatisticsKHR; - } - pipelineFlags2CreateInfo.setPNext(compute_pipeline_create_info.pNext); - compute_pipeline_create_info.setPNext(&pipelineFlags2CreateInfo); - } -#endif - - try { - pipeline->pipeline = device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value; - } catch (const vk::SystemError& e) { - std::cerr << "ggml_vulkan: Compute pipeline creation failed for " << pipeline->name << std::endl; - std::cerr << "ggml_vulkan: " << e.what() << std::endl; - throw e; - } - - if (vk_instance.debug_utils_support) { - vk::DebugUtilsObjectNameInfoEXT duoni; - duoni.objectType = vk::ObjectType::ePipeline; - duoni.pObjectName = pipeline->name.c_str(); - duoni.objectHandle = /*reinterpret_cast*/(uint64_t)(static_cast<VkPipeline>(pipeline->pipeline)); - vk_instance.pfn_vkSetDebugUtilsObjectNameEXT(device->device, &static_cast<VkDebugUtilsObjectNameInfoEXT &>(duoni)); - } - - if (device->pipeline_executable_properties_support) { - vk::PipelineExecutableInfoKHR executableInfo; - executableInfo.pipeline = pipeline->pipeline; - - auto statistics = device->device.getPipelineExecutableStatisticsKHR(executableInfo); - - bool print_stats = !vk_pipeline_stats_filter.empty() && - pipeline->name.find(vk_pipeline_stats_filter) != std::string::npos; - if (print_stats) { - std::cerr << "ggml_vulkan: pipeline stats for " << pipeline->name << ":" << std::endl; - } + vk_device_architecture::AMD_RDNA1, + { + rdna1_pipelines, + }, + RDNA_DEFAULT_SUBGROUP_SIZE + }, + { + vk_device_architecture::AMD_RDNA2, + { + rdna2_pipelines, + }, + RDNA_DEFAULT_SUBGROUP_SIZE + }, +}; - for (auto & s : statistics) { - if (print_stats) { - std::cerr << "ggml_vulkan: " << s.name.data() << ": "; - switch (s.format) { - case vk::PipelineExecutableStatisticFormatKHR::eBool32: - std::cerr << (s.value.b32 ? "true" : "false"); - break; - case vk::PipelineExecutableStatisticFormatKHR::eInt64: - std::cerr << s.value.i64; - break; - case vk::PipelineExecutableStatisticFormatKHR::eUint64: - std::cerr << s.value.u64; - break; - case vk::PipelineExecutableStatisticFormatKHR::eFloat64: - std::cerr << s.value.f64; - break; - } - std::cerr << std::endl; +uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch) { + for (const auto &config : gpu_pipeline_configs) { + if (config.arch == arch) { + auto pipIt = config.pipelines.find(pipeline_name); + if (pipIt != config.pipelines.end()) { + return pipIt->second; } - // "Register Count" is reported by NVIDIA drivers. - if (strcmp(s.name, "Register Count") == 0) { - VK_LOG_DEBUG(pipeline->name << " " << s.name << ": " << s.value.u64 << " registers"); - pipeline->register_count = (uint32_t)s.value.u64; + std::vector<std::pair<std::string, uint32_t>> sorted_pipelines(config.pipelines.begin(), config.pipelines.end()); + std::sort(sorted_pipelines.begin(), sorted_pipelines.end(), + [](const auto &a, const auto &b) { return a.first.size() > b.first.size(); }); + for (const auto &entry : sorted_pipelines) { + if (pipeline_name.find(entry.first) != std::string::npos) { + return entry.second; + } } + return config.default_subgroup_size; } } - - { - std::lock_guard<std::mutex> guard(device->compile_mutex); - device->all_pipelines.push_back(pipeline); - pipeline->compiled = true; - pipeline->compile_pending = false; - } - device->compile_cv.notify_all(); + return 0; // If no matching configuration is found } -static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline) { - VK_LOG_DEBUG("ggml_pipeline_destroy_pipeline(" << pipeline->name << ")"); - device.destroyPipelineLayout(pipeline->layout); - - device.destroyShaderModule(pipeline->shader_module); - - device.destroyPipeline(pipeline->pipeline); +static bool ggml_vk_fa_type_needs_shmem(ggml_type type) { + switch (type) { + case GGML_TYPE_IQ4_NL: + return true; + default: + return false; + } } -static void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, vk_pipeline& pipeline, uint32_t n) { - VK_LOG_DEBUG("ggml_pipeline_request_descriptor_sets(" << pipeline->name << ", " << n << ")"); - ggml_vk_pipeline_descriptor_set_requirements(ctx) += n; - if (!pipeline->compiled) { - ggml_vk_load_shaders(ctx->device, pipeline); - } - ggml_pipeline_allocate_descriptor_sets(ctx); +static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type, ggml_type v_type) { +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + return device->integer_dot_product && device->subgroup_clustered && + !ggml_vk_fa_type_needs_shmem(v_type) && + (k_type == GGML_TYPE_Q4_0 || k_type == GGML_TYPE_Q4_1 || + k_type == GGML_TYPE_Q5_0 || k_type == GGML_TYPE_Q5_1 || + k_type == GGML_TYPE_Q8_0); +#else + GGML_UNUSED(device); + GGML_UNUSED(k_type); + GGML_UNUSED(v_type); + return false; +#endif } -static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx) { - std::vector<vk::DescriptorPool> & descriptor_pools = ggml_vk_descriptor_pools(ctx); - std::vector<vk::DescriptorSet> & descriptor_sets = ggml_vk_descriptor_sets(ctx); - const uint32_t descriptor_set_requirements = - ggml_vk_pipeline_descriptor_set_requirements(ctx); +void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { + VK_LOG_DEBUG("ggml_vk_load_shaders(" << device->name << ")"); - if (descriptor_sets.size() >= descriptor_set_requirements) { - // Enough descriptors are available - return; - } + // some shaders have a minimum subgroup size + const uint32_t subgroup_size_8 = std::max(device->subgroup_size, 8u); + const uint32_t subgroup_size_16 = std::max(device->subgroup_size, 16u); + const uint32_t subgroup_size_32 = std::max(device->subgroup_size, 32u); - vk_device& device = ctx->device; + // clamp WARP for l_/m_ warptiles so WM <= BM (breaks on subgroupSize > 64) + const uint32_t mm_warp_8 = std::min(subgroup_size_8, 64u); + const uint32_t mm_warp_16 = std::min(subgroup_size_16, 64u); - // Grow by 50% to avoid frequent allocations - uint32_t needed = std::max(3 * descriptor_sets.size() / 2, size_t{descriptor_set_requirements}); - uint32_t to_alloc = needed - descriptor_sets.size(); - uint32_t pool_remaining = VK_DEVICE_DESCRIPTOR_POOL_SIZE - descriptor_sets.size() % VK_DEVICE_DESCRIPTOR_POOL_SIZE; - uint32_t pool_idx = descriptor_sets.size() / VK_DEVICE_DESCRIPTOR_POOL_SIZE; + const uint32_t mul_mat_subgroup_size = (device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control) ? device->subgroup_min_size : device->subgroup_size; + const uint32_t mul_mat_subgroup_size_8 = std::max(mul_mat_subgroup_size, 8u); + const uint32_t mul_mat_subgroup_size_16 = std::max(mul_mat_subgroup_size, 16u); + const uint32_t mul_mat_subgroup_size_32 = std::max(mul_mat_subgroup_size, 32u); + const uint32_t mul_mat_mm_warp_8 = std::min(mul_mat_subgroup_size_8, 64u); + const uint32_t mul_mat_mm_warp_16 = std::min(mul_mat_subgroup_size_16, 64u); - while (to_alloc > 0) { - const uint32_t alloc_count = std::min(pool_remaining, to_alloc); - to_alloc -= alloc_count; - pool_remaining = VK_DEVICE_DESCRIPTOR_POOL_SIZE; + const bool subgroup_min_size_16 = (!device->subgroup_size_control && device->subgroup_size >= 16) || + (device->subgroup_size_control && device->subgroup_max_size >= 16); - if (pool_idx >= descriptor_pools.size()) { - vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, MAX_PARAMETER_COUNT * VK_DEVICE_DESCRIPTOR_POOL_SIZE); - vk::DescriptorPoolCreateInfo descriptor_pool_create_info({}, VK_DEVICE_DESCRIPTOR_POOL_SIZE, descriptor_pool_size); - descriptor_pools.push_back(device->device.createDescriptorPool(descriptor_pool_create_info)); - } + // mulmat + // Warptile layout (indices match mul_mm.comp constantIDs): + // [0..9] : BLOCK_SIZE, BM, BN, BK, WM, WN, WMITER, TM, TN, TK + // [10] : WARP / required_subgroup_size (read via WARP_SIZE_IDX) + static constexpr size_t WARP_SIZE_IDX = 10; + std::vector<uint32_t> l_warptile, m_warptile, s_warptile, + l_warptile_id, m_warptile_id, s_warptile_id, + l_warptile_mmq, m_warptile_mmq, s_warptile_mmq, + l_warptile_mmq_int, m_warptile_mmq_int, s_warptile_mmq_int, + l_warptile_mmq_int_k, m_warptile_mmq_int_k, s_warptile_mmq_int_k, + l_warptile_mmq_k, m_warptile_mmq_k, s_warptile_mmq_k, + l_warptile_mmqid, m_warptile_mmqid, s_warptile_mmqid, + l_warptile_mmqid_int, m_warptile_mmqid_int, s_warptile_mmqid_int, + l_warptile_mmqid_int_k, m_warptile_mmqid_int_k, s_warptile_mmqid_int_k; + std::array<uint32_t, 3> l_wg_denoms, m_wg_denoms, s_wg_denoms, + l_mmq_wg_denoms, m_mmq_wg_denoms, s_mmq_wg_denoms, + l_mmq_wg_denoms_k, m_mmq_wg_denoms_k, s_mmq_wg_denoms_k, + l_mmqid_wg_denoms, m_mmqid_wg_denoms, s_mmqid_wg_denoms; - std::vector<vk::DescriptorSetLayout> layouts(alloc_count); - for (uint32_t i = 0; i < alloc_count; i++) { - layouts[i] = device->dsl; - } - vk::DescriptorSetAllocateInfo descriptor_set_alloc_info(descriptor_pools[pool_idx], alloc_count, layouts.data()); - std::vector<vk::DescriptorSet> sets = device->device.allocateDescriptorSets(descriptor_set_alloc_info); - descriptor_sets.insert(descriptor_sets.end(), sets.begin(), sets.end()); + uint32_t l_align, m_align, s_align; - pool_idx++; - } -} + vk_pipeline wait_pipeline; + CompileTask claimed_task {}; + bool has_claimed_task = false; -static vk_command_buffer* ggml_vk_create_cmd_buffer(vk_device& device, vk_command_pool& p) { - VK_LOG_DEBUG("ggml_vk_create_cmd_buffer()"); - vk::CommandBufferAllocateInfo command_buffer_alloc_info( - p.pool, - vk::CommandBufferLevel::ePrimary, - 1); - const std::vector<vk::CommandBuffer> cmd_buffers = device->device.allocateCommandBuffers(command_buffer_alloc_info); - p.cmd_buffers.push_back({ cmd_buffers.front(), 0, true }); - return &p.cmd_buffers[p.cmd_buffers.size()-1]; -} + // The rest of the walk reads and writes shared device state, so hold the + // lock until we're done deciding what to compile. + std::unique_lock<std::mutex> compile_lock(device->compile_mutex); -static void ggml_vk_submit(vk_context& ctx, vk::Fence fence) { - if (ctx->seqs.empty()) { - if (fence) { - ctx->p->q->handle->submit({}, fence); - } - return; - } - VK_LOG_DEBUG("ggml_vk_submit(" << ctx << ", " << fence << ")"); + if (device->coopmat2) { + // spec constants and tile sizes for non-quant matmul/matmul_id + l_warptile = { 256, 128, 256, 64, 1 }; + m_warptile = { 256, 128, 128, 64, 0 }; + s_warptile = { 128, 64, 64, 64, 0 }; + l_wg_denoms = {128, 256, 1 }; + m_wg_denoms = {128, 128, 1 }; + s_wg_denoms = { 64, 64, 1 }; - std::vector<std::vector<uint64_t>> tl_wait_vals; - std::vector<std::vector<uint64_t>> tl_signal_vals; - std::vector<std::vector<vk::Semaphore>> tl_wait_semaphores; - std::vector<std::vector<vk::Semaphore>> tl_signal_semaphores; - std::vector<vk::TimelineSemaphoreSubmitInfo> tl_submit_infos; - std::vector<vk::SubmitInfo> submit_infos; - int idx = -1; - std::vector<std::vector<vk::PipelineStageFlags>> stage_flags; + // spec constants and tile sizes for quant matmul (non-Qi_K) + l_warptile_mmq = { 256, 128, 256, 64, 1 }; + m_warptile_mmq = { 256, 128, 128, 64, 1 }; + s_warptile_mmq = { 256, 32, 64, 128, 0 }; + l_mmq_wg_denoms = { 128, 256, 1 }; + m_mmq_wg_denoms = { 128, 128, 1 }; + s_mmq_wg_denoms = { 32, 64, 1 }; - size_t reserve = 0; + // spec constants and tile sizes for quant matmul (Qi_K) + l_warptile_mmq_k = { 256, 128, 256, 64, 1 }; + m_warptile_mmq_k = { 256, 128, 128, 64, 1 }; + s_warptile_mmq_k = { 256, 32, 64, 128, 0 }; + l_mmq_wg_denoms_k = { 128, 256, 1 }; + m_mmq_wg_denoms_k = { 128, 128, 1 }; + s_mmq_wg_denoms_k = { 32, 64, 1 }; - for (const auto& sequence : ctx->seqs) { - reserve += sequence.size(); - } + // spec constants and tile sizes for quant matmul_id + const uint32_t mmqid_bk = device->coopmat2_decode_vector ? 64u : 32u; + l_warptile_mmqid = { 256, 128, 128, mmqid_bk, 1 }; + m_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0 }; + s_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0 }; + l_mmqid_wg_denoms = { 128, 128, 1 }; + m_mmqid_wg_denoms = { 128, 64, 1 }; + s_mmqid_wg_denoms = { 128, 64, 1 }; - // Pre-reserve vectors to prevent reallocation, which invalidates pointers - tl_wait_semaphores.reserve(reserve); - tl_wait_vals.reserve(reserve); - tl_signal_semaphores.reserve(reserve); - tl_signal_vals.reserve(reserve); - tl_submit_infos.reserve(reserve); - submit_infos.reserve(reserve); - stage_flags.reserve(reserve); - - for (const auto& sequence : ctx->seqs) { - for (const auto& submission : sequence) { - stage_flags.push_back({}); - idx++; - tl_wait_vals.push_back({}); - tl_wait_semaphores.push_back({}); - tl_signal_vals.push_back({}); - tl_signal_semaphores.push_back({}); - for (size_t i = 0; i < submission.wait_semaphores.size(); i++) { - stage_flags[idx].push_back(ctx->p->q->stage_flags); - tl_wait_vals[idx].push_back(submission.wait_semaphores[i].value); - tl_wait_semaphores[idx].push_back(submission.wait_semaphores[i].s); - } - for (size_t i = 0; i < submission.signal_semaphores.size(); i++) { - tl_signal_vals[idx].push_back(submission.signal_semaphores[i].value); - tl_signal_semaphores[idx].push_back(submission.signal_semaphores[i].s); - } - tl_submit_infos.push_back({ - (uint32_t) submission.wait_semaphores.size(), - tl_wait_vals[idx].data(), - (uint32_t) submission.signal_semaphores.size(), - tl_signal_vals[idx].data(), - }); - tl_submit_infos[idx].sType = vk::StructureType::eTimelineSemaphoreSubmitInfo; - tl_submit_infos[idx].pNext = nullptr; - vk::SubmitInfo si{ - (uint32_t) submission.wait_semaphores.size(), - tl_wait_semaphores[idx].data(), - stage_flags[idx].data(), - 1, - &submission.buffer->buf, - (uint32_t) submission.signal_semaphores.size(), - tl_signal_semaphores[idx].data(), - }; - si.setPNext(&tl_submit_infos[idx]); - submit_infos.push_back(si); - } - } + l_align = 128; + m_align = 64; + s_align = 32; + } else { + // Matrix cores require different warp group sizes + const uint32_t tm_l = device->coopmat_support ? device->coopmat_m : 4; + const uint32_t tm_m = device->coopmat_support ? device->coopmat_m : 4; + const uint32_t tm_s = device->coopmat_support ? device->coopmat_m : 2; + const uint32_t tn_l = device->coopmat_support ? device->coopmat_n : 4; + const uint32_t tn_m = device->coopmat_support ? device->coopmat_n : 2; + const uint32_t tn_s = device->coopmat_support ? device->coopmat_n : 2; + const uint32_t tk_l = device->coopmat_support ? device->coopmat_k : 1; + const uint32_t tk_m = device->coopmat_support ? device->coopmat_k : 1; + const uint32_t tk_s = device->coopmat_support ? device->coopmat_k : 1; - ctx->p->q->handle->submit(submit_infos, fence); + const uint32_t s_warptile_wm = device->subgroup_size == 8 ? 8 : 32; - ctx->seqs.clear(); -} + l_warptile = { 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, tm_l, tn_l, tk_l, mm_warp_8 }; + m_warptile = { 128, 64, 64, 16, mm_warp_8, 32, 2, tm_m, tn_m, tk_m, mm_warp_8 }; + s_warptile = { subgroup_size_32, 32, 32, 16, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, subgroup_size_8 }; -static uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyProperties>& queue_family_props, const vk::QueueFlags& required, const vk::QueueFlags& avoid, int32_t compute_index, uint32_t min_num_queues) { - VK_LOG_DEBUG("ggml_vk_find_queue_family_index()"); - const uint32_t qfsize = queue_family_props.size(); + l_warptile_mmq = { 128, 128, 128, 32, mm_warp_8 * 2, 64, 2, tm_l, tn_l, tk_l, mm_warp_8 }; + m_warptile_mmq = { 128, 64, 64, 32, mm_warp_8, 32, 2, tm_m, tn_m, tk_m, mm_warp_8 }; + s_warptile_mmq = { subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, subgroup_size_8 }; - // Try with avoid preferences first - for (uint32_t i = 0; i < qfsize; i++) { - if (queue_family_props[i].queueCount >= min_num_queues && (compute_index < 0 || i != (uint32_t) compute_index) && queue_family_props[i].queueFlags & required && !(queue_family_props[i].queueFlags & avoid)) { - return i; - } - } + // Integer MMQ has a smaller shared memory profile, but heavier register use + l_warptile_mmq_int = { 128, 128, 128, 32, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 }; + m_warptile_mmq_int = { 128, 64, 64, 32, mm_warp_8, 32, 2, 2, 2, 1, mm_warp_8 }; + s_warptile_mmq_int = { subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, 2, 1, 1, subgroup_size_8 }; - // Fall back to only required - for (size_t i = 0; i < qfsize; i++) { - if (queue_family_props[i].queueCount >= min_num_queues && (compute_index < 0 || i != (uint32_t) compute_index) && queue_family_props[i].queueFlags & required) { - return i; - } - } + // K-quants use even more registers, mitigate by setting WMITER to 1 + l_warptile_mmq_int_k = { 128, 128, 128, 32, mm_warp_8 * 2, 64, 1, 4, 4, 1, mm_warp_8 }; + m_warptile_mmq_int_k = { 128, 64, 64, 32, mm_warp_8, 32, 1, 2, 2, 1, mm_warp_8 }; + s_warptile_mmq_int_k = { subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 1, 2, 1, 1, subgroup_size_8 }; - // Fall back to reusing compute queue - for (size_t i = 0; i < qfsize; i++) { - if (queue_family_props[i].queueCount >= min_num_queues && queue_family_props[i].queueFlags & required) { - return i; - } - } + l_warptile_id = { 128, 128, 128, 16, mul_mat_mm_warp_16 * 2, 64, 2, tm_l, tn_l, tk_l, mul_mat_mm_warp_16 }; + m_warptile_id = { 128, 64, 64, 16, mul_mat_mm_warp_16, 32, 2, tm_m, tn_m, tk_m, mul_mat_mm_warp_16 }; + s_warptile_id = { mul_mat_subgroup_size_16, 32, 32, 16, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, mul_mat_subgroup_size_16 }; - // Fall back to ignoring min_num_queries - for (size_t i = 0; i < qfsize; i++) { - if (queue_family_props[i].queueFlags & required) { - return i; - } - } + l_warptile_mmqid = { 128, 128, 128, 32, mul_mat_mm_warp_8 * 2, 64, 2, tm_l, tn_l, tk_l, mul_mat_mm_warp_8 }; + m_warptile_mmqid = { 128, 64, 64, 32, mul_mat_mm_warp_8, 32, 2, tm_m, tn_m, tk_m, mul_mat_mm_warp_8 }; + s_warptile_mmqid = { mul_mat_subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, mul_mat_subgroup_size_8 }; - // All commands that are allowed on a queue that supports transfer operations are also allowed on a queue that supports either graphics or compute operations. - // Thus, if the capabilities of a queue family include VK_QUEUE_GRAPHICS_BIT or VK_QUEUE_COMPUTE_BIT, then reporting the VK_QUEUE_TRANSFER_BIT capability separately for that queue family is optional. - if (compute_index >= 0) { - return compute_index; - } + l_warptile_mmqid_int = { 128, 128, 128, 32, mul_mat_mm_warp_8 * 2, 64, 2, 4, 4, 1, mul_mat_mm_warp_8 }; + m_warptile_mmqid_int = { 128, 64, 64, 32, mul_mat_mm_warp_8, 32, 2, 2, 2, 1, mul_mat_mm_warp_8 }; + s_warptile_mmqid_int = { mul_mat_subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, 2, 1, 1, mul_mat_subgroup_size_8 }; - std::cerr << "ggml_vulkan: No suitable queue family index found." << std::endl; + l_warptile_mmqid_int_k = { 128, 128, 128, 32, mul_mat_mm_warp_16 * 2, 64, 1, 4, 4, 1, mul_mat_mm_warp_16 }; + m_warptile_mmqid_int_k = { 128, 64, 64, 32, mul_mat_mm_warp_16, 32, 1, 2, 2, 1, mul_mat_mm_warp_16 }; + s_warptile_mmqid_int_k = { mul_mat_subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 1, 2, 1, 1, mul_mat_subgroup_size_16 }; - for(auto &q_family : queue_family_props) { - std::cerr << "Queue number: " + std::to_string(q_family.queueCount) << " flags: " + to_string(q_family.queueFlags) << std::endl; - } - abort(); -} + // chip specific tuning + if ((device->architecture == AMD_GCN) && (device->driver_id != vk::DriverId::eAmdProprietary)) { + m_warptile_mmq = m_warptile_mmq_int = { 256, 64, 64, 32, 16, 16, 2, 2, 2, 1, 16 }; + m_warptile_mmqid = m_warptile_mmqid_int = { 256, 64, 64, 32, 16, 16, 2, 2, 2, 1, 16 }; + } else if (device->vendor_id == VK_VENDOR_ID_AMD && device->coopmat_support && device->driver_id != vk::DriverId::eAmdProprietary) { + // This is intentionally using tx_m values, slight performance increase + l_warptile = { 256, 128, 128, 16, mm_warp_8, 64, 2, tm_m, tn_m, tk_m, mm_warp_8 }; + l_warptile_mmq = l_warptile_mmq_int = { 256, 128, 128, 32, mm_warp_8, 64, 2, tm_m, tn_m, tk_m, mm_warp_8 }; + l_warptile_mmq_int_k = { 256, 128, 128, 32, mm_warp_16, 64, 1, 4, 2, 1, mm_warp_16 }; + } -static std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_t queue_family_index, uint32_t queue_index, vk::PipelineStageFlags&& stage_flags, bool transfer_only) { - VK_LOG_DEBUG("ggml_vk_create_queue()"); - std::lock_guard<std::recursive_mutex> guard(device->mutex); + l_mmq_wg_denoms = l_wg_denoms = {128, 128, 1 }; + m_mmq_wg_denoms = m_wg_denoms = { 64, 64, 1 }; + s_mmq_wg_denoms = s_wg_denoms = { 32, 32, 1 }; + l_align = 128; + m_align = 64; + s_align = 32; - auto q = std::make_unique<vk_queue>(); - q->queue_family_index = queue_family_index; - q->transfer_only = transfer_only; + if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support) { + // Xe1/Xe2/Xe3 with coopmat enabled - warptile performance tuning + l_warptile = { 512, 128, 128, 16, mm_warp_8, 32, 2, tm_l, tn_l, tk_l, mm_warp_8 }; + if (device->architecture == INTEL_XE1) { + l_warptile_mmq = { 512, 256, 128, 32, 32, 32, 2, tm_l, tn_l, tk_l, 16 }; + l_mmq_wg_denoms = { 256, 128, 1 }; + l_align = 32; //set as BK + } else { + l_warptile_mmq = { 512, 128, 256, 32, 32, 32, 2, tm_l, tn_l, tk_l, 16 }; + l_mmq_wg_denoms = { 128, 256, 1 }; + l_align = 32; //set as BK + } + } - std::shared_ptr<vk_queue_handle> h; - vk::DeviceQueueInfo2 queue_info2{}; - queue_info2.queueFamilyIndex = queue_family_index; - queue_info2.queueIndex = queue_index; + for (uint32_t i = 0; i < GGML_TYPE_COUNT; ++i) { + ggml_type t = (ggml_type)i; + // Disable medium and large matrix multiplication if not enough shared memory is available + // Check mmq warptiles as the largest configuration + // Throw an error if not enough for any matrix multiplication is available + if (!ggml_vk_matmul_shmem_support(device, s_warptile_mmq, false, t)) { + std::cerr << "ggml_vulkan: Error: Shared memory size too small for matrix multiplication." << std::endl; + throw std::runtime_error("Shared memory size too small for matrix multiplication."); + } else if (!ggml_vk_matmul_shmem_support(device, m_warptile_mmq, false, t)) { + device->mul_mat_m[i] = false; + device->mul_mat_l[i] = false; + } else if (!ggml_vk_matmul_shmem_support(device, l_warptile_mmq, false, t)) { + device->mul_mat_l[i] = false; + } - if (device->has_internally_synchronized_queues) { - h = std::make_shared<vk_queue_handle_unsynchronized>(); - queue_info2.flags = eInternallySynchronizedKHR; - } else { - h = std::make_shared<vk_queue_handle_synchronized>(); - } + // Disable mul_mat_id if not enough shared memory is available + if (!ggml_vk_matmul_shmem_support(device, s_warptile_mmqid, true, t)) { + device->mul_mat_id_s[i] = false; + device->mul_mat_id_m[i] = false; + device->mul_mat_id_l[i] = false; + } else if (!ggml_vk_matmul_shmem_support(device, m_warptile_mmqid, true, t)) { + device->mul_mat_id_m[i] = false; + device->mul_mat_id_l[i] = false; + } else if (!ggml_vk_matmul_shmem_support(device, l_warptile_mmqid, true, t)) { + device->mul_mat_id_l[i] = false; + } - h->queue = device->device.getQueue2(queue_info2); - h->device = device; - q->handle = h; + // The q8_1 mmq path has its own (larger) shmem layout, check it separately. + // K-quants and IQ3_S use the _int_k warptiles, others use _int. + const bool is_k_quant = (t == GGML_TYPE_Q2_K || t == GGML_TYPE_Q3_K || + t == GGML_TYPE_Q4_K || t == GGML_TYPE_Q5_K || + t == GGML_TYPE_Q6_K || t == GGML_TYPE_IQ3_S); + const auto & s_int = is_k_quant ? s_warptile_mmq_int_k : s_warptile_mmq_int; + const auto & m_int = is_k_quant ? m_warptile_mmq_int_k : m_warptile_mmq_int; + const auto & l_int = is_k_quant ? l_warptile_mmq_int_k : l_warptile_mmq_int; + const auto & s_intid = is_k_quant ? s_warptile_mmqid_int_k : s_warptile_mmqid_int; + const auto & m_intid = is_k_quant ? m_warptile_mmqid_int_k : m_warptile_mmqid_int; + const auto & l_intid = is_k_quant ? l_warptile_mmqid_int_k : l_warptile_mmqid_int; - q->cmd_pool.init(device, q.get()); + if (!ggml_vk_matmul_int_shmem_support(device, s_int, false, t)) { + device->mul_mat_s_int[i] = false; + device->mul_mat_m_int[i] = false; + device->mul_mat_l_int[i] = false; + } else if (!ggml_vk_matmul_int_shmem_support(device, m_int, false, t)) { + device->mul_mat_m_int[i] = false; + device->mul_mat_l_int[i] = false; + } else if (!ggml_vk_matmul_int_shmem_support(device, l_int, false, t)) { + device->mul_mat_l_int[i] = false; + } - q->stage_flags = stage_flags; - return q; -} + if (!ggml_vk_matmul_int_shmem_support(device, s_intid, true, t)) { + device->mul_mat_id_s_int[i] = false; + device->mul_mat_id_m_int[i] = false; + device->mul_mat_id_l_int[i] = false; + } else if (!ggml_vk_matmul_int_shmem_support(device, m_intid, true, t)) { + device->mul_mat_id_m_int[i] = false; + device->mul_mat_id_l_int[i] = false; + } else if (!ggml_vk_matmul_int_shmem_support(device, l_intid, true, t)) { + device->mul_mat_id_l_int[i] = false; + } + } + } -static std::unique_ptr<vk_queue> ggml_vk_create_aliased_queue(vk_device& device, const std::unique_ptr<vk_queue>& source) { - std::lock_guard<std::recursive_mutex> guard(device->mutex); - auto q = std::make_unique<vk_queue>(); - q->handle = source->handle; - q->queue_family_index = source->queue_family_index; - q->stage_flags = source->stage_flags; - q->transfer_only = source->transfer_only; - q->cmd_pool.init(device, q.get()); - return q; -} + auto const &ggml_vk_create_pipeline = [&](vk_device& device, vk_pipeline& base_pipeline, const char *name, size_t spv_size, const void* spv_data, const char *entrypoint, + uint32_t parameter_count, uint32_t push_constant_size, std::array<uint32_t, 3> wg_denoms, const std::vector<uint32_t>& specialization_constants, + uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) { -static vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_command_pool& p) { - vk_context result = std::make_shared<vk_context_struct>(); - VK_LOG_DEBUG("ggml_vk_create_context(" << result << ")"); - ctx->gc.contexts.emplace_back(result); - result->p = &p; - return result; -} + if (!require_full_subgroups && required_subgroup_size == 0) { + required_subgroup_size = get_subgroup_size(name, device->architecture); + } -static vk_context ggml_vk_create_temporary_context(vk_command_pool& p) { - vk_context result = std::make_shared<vk_context_struct>(); - VK_LOG_DEBUG("ggml_vk_create_temporary_context(" << result << ")"); - result->p = &p; - return result; -} + vk_pipeline *ptr = &base_pipeline; -static vk_semaphore * ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx) { - VK_LOG_DEBUG("ggml_vk_create_timeline_semaphore()"); - vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eBinary, 0 }; - vk::SemaphoreCreateInfo ci{}; - ci.setPNext(&tci); - vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci); - ctx->gc.semaphores.push_back({ semaphore, 0 }); - return &ctx->gc.semaphores[ctx->gc.semaphores.size() - 1]; -} - -static vk_semaphore * ggml_vk_create_timeline_semaphore(ggml_backend_vk_context * ctx) { - VK_LOG_DEBUG("ggml_vk_create_timeline_semaphore()"); - if (ctx->semaphore_idx >= ctx->gc.tl_semaphores.size()) { - vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eTimeline, 0 }; - vk::SemaphoreCreateInfo ci{}; - ci.setPNext(&tci); - vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci); - ctx->gc.tl_semaphores.push_back({ semaphore, 0 }); - } - return &ctx->gc.tl_semaphores[ctx->semaphore_idx++]; -} + int num_pipelines = 1; +#if defined(VK_EXT_shader_64bit_indexing) + if (device->shader_64b_indexing) { + num_pipelines = 2; + } +#endif + for (int i = 0; i < num_pipelines; ++i, ptr = &(*ptr)->next) { + vk_pipeline &pipeline = *ptr; + if (!pipeline) { + pipeline = std::make_shared<vk_pipeline_struct>(); + } + if (!pipeline->initialized) { + pipeline->name = name; + pipeline->parameter_count = parameter_count; + pipeline->push_constant_size = push_constant_size; + pipeline->wg_denoms = wg_denoms; + pipeline->align = align; + pipeline->initialized = true; +#if defined(VK_EXT_shader_64bit_indexing) + pipeline->is_64b_indexing = (i == 1); +#endif + } -static vk::Event ggml_vk_create_event(ggml_backend_vk_context * ctx) { - if (ctx->event_idx >= ctx->gc.events.size()) { - ctx->gc.events.push_back(ctx->device->device.createEvent({})); - } - return ctx->gc.events[ctx->event_idx++]; -} + // We only care about the pipeline this call asked for; the rest + // (including the 64-bit indexing variant) are handled by their + // own request_descriptor_sets / load_shaders calls. + if (pipeline.get() != requested.get()) { + continue; + } -static void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p) { - VK_LOG_DEBUG("ggml_vk_command_pool_cleanup()"); + if (pipeline->compiled) { + continue; + } - // Requires command buffers to be done - device->device.resetCommandPool(p.pool); - // Don't clear the command buffers and mark them as not in use. - // This allows us to reuse them - for (auto& cmd_buffer : p.cmd_buffers) { - cmd_buffer.in_use = false; - } -} + wait_pipeline = pipeline; -static void ggml_vk_queue_command_pools_cleanup(vk_device& device) { - VK_LOG_DEBUG("ggml_vk_queue_command_pools_cleanup()"); + if (!pipeline->compile_pending) { + pipeline->compile_pending = true; + claimed_task.pipeline = pipeline; + claimed_task.spv_size = spv_size; + claimed_task.spv_data = spv_data; + claimed_task.entrypoint = entrypoint; + claimed_task.parameter_count = parameter_count; + claimed_task.wg_denoms = wg_denoms; + claimed_task.specialization_constants = specialization_constants; + claimed_task.disable_robustness = disable_robustness; + claimed_task.require_full_subgroups = require_full_subgroups; + claimed_task.required_subgroup_size = required_subgroup_size; + has_claimed_task = true; + } + } + }; - // Arbitrary frequency to cleanup/reuse command buffers - static constexpr uint32_t cleanup_frequency = 10; + auto const &ggml_vk_create_pipeline2 = [&](vk_device& device, vk_pipeline& pipeline, const std::string &name, size_t spv_size, const void* spv_data, const char *entrypoint, + uint32_t parameter_count, uint32_t push_constant_size, std::array<uint32_t, 3> wg_denoms, const std::vector<uint32_t>& specialization_constants, + uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) { + return ggml_vk_create_pipeline(device, pipeline, name.c_str(), spv_size, spv_data, entrypoint, + parameter_count, push_constant_size, wg_denoms, specialization_constants, + align, disable_robustness, require_full_subgroups, required_subgroup_size); + }; - if (device->compute_queue && device->compute_queue->cmd_pool.buffers_in_use() >= cleanup_frequency) { - ggml_vk_command_pool_cleanup(device, device->compute_queue->cmd_pool); - } - if (device->transfer_queue && device->transfer_queue->cmd_pool.buffers_in_use() >= cleanup_frequency) { - ggml_vk_command_pool_cleanup(device, device->transfer_queue->cmd_pool); - } -} + // FA scalar has two SPIR-V modules (MMQ vs non-MMQ); FA cm1 has one. K/V + // quant type is selected at runtime via the FaTypeK / FaTypeV spec constants. -static std::vector<uint32_t> ggml_vk_find_memory_properties(const vk::PhysicalDeviceMemoryProperties* mem_props, vk::MemoryRequirements* mem_req, vk::MemoryPropertyFlags flags) { - std::vector<uint32_t> indices; + for (auto &fa : device->pipeline_flash_attn_f32_f16) { + if (fa.first.path != FA_SCALAR) continue; + const uint32_t Br = fa.first.Br; + const uint32_t Bc = fa.first.Bc; + const bool aligned = fa.first.aligned; + const bool f32acc = fa.first.f32acc; + const uint32_t fa_sgs = fa.first.subgroup_size; + const bool fa_ds = fa.first.subgroup_size == 0; - for (uint32_t i = 0; i < mem_props->memoryTypeCount; ++i) { - vk::MemoryType memory_type = mem_props->memoryTypes[i]; - if ((mem_req->memoryTypeBits & ((uint64_t)1 << i)) && - (flags & memory_type.propertyFlags) == flags && - mem_props->memoryHeaps[memory_type.heapIndex].size >= mem_req->size) { - indices.push_back(i); + const bool bf16_kv = fa.first.k_type == GGML_TYPE_BF16; + const bool use_mmq = ggml_vk_fa_scalar_uses_mmq(device, fa.first.k_type, fa.first.v_type); + const void * spv_data = nullptr; + size_t spv_size = 0; + const char *name = nullptr; + if (bf16_kv) { + spv_data = flash_attn_f32_f16_fp32_data; + spv_size = flash_attn_f32_f16_fp32_len; + name = aligned ? "flash_attn_f32_bf16_aligned" : "flash_attn_f32_bf16"; + } else if (use_mmq) { +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (device->fp16) { + if (f32acc) { spv_data = flash_attn_f32_f16_int8_data; spv_size = flash_attn_f32_f16_int8_len; } + else { spv_data = flash_attn_f32_f16_f16acc_int8_data; spv_size = flash_attn_f32_f16_f16acc_int8_len; } + } else { + spv_data = flash_attn_f32_f16_fp32_int8_data; + spv_size = flash_attn_f32_f16_fp32_int8_len; + } +#endif + name = aligned ? "flash_attn_f32_f16_aligned" : "flash_attn_f32_f16"; + } else { + if (device->fp16) { + if (device->dot2_f16) { + if (f32acc) { spv_data = flash_attn_f32_f16_dot2_data; spv_size = flash_attn_f32_f16_dot2_len; } + else { spv_data = flash_attn_f32_f16_dot2_f16acc_data; spv_size = flash_attn_f32_f16_dot2_f16acc_len; } + } else { + if (f32acc) { spv_data = flash_attn_f32_f16_data; spv_size = flash_attn_f32_f16_len; } + else { spv_data = flash_attn_f32_f16_f16acc_data; spv_size = flash_attn_f32_f16_f16acc_len; } + } + } else { + spv_data = flash_attn_f32_f16_fp32_data; + spv_size = flash_attn_f32_f16_fp32_len; + } + name = aligned ? "flash_attn_f32_f16_aligned" : "flash_attn_f32_f16"; } - } - return indices; -} - -static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, const std::initializer_list<vk::MemoryPropertyFlags> & req_flags_list, - void *import_ptr = nullptr) { - VK_LOG_DEBUG("ggml_vk_create_buffer(" << device->name << ", " << size << ", " << to_string(req_flags_list.begin()[0]) << ", " << to_string(req_flags_list.begin()[req_flags_list.size()-1]) << ")"); - if (size > device->max_buffer_size) { - throw vk::OutOfDeviceMemoryError("Requested buffer size exceeds device buffer size limit"); + ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 8, + sizeof(vk_flash_attn_push_constants), {Br, 1, 1}, + get_fa_spec_constants(fa.first), aligned ? Bc : 1, true, + !fa_ds, !fa_ds ? fa_sgs : 0); } - vk_buffer buf = std::make_shared<vk_buffer_struct>(); +#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + if (device->coopmat1_fa_support) { + for (auto &fa : device->pipeline_flash_attn_f32_f16) { + if (fa.first.path != FA_COOPMAT1) continue; + const uint32_t Br = fa.first.Br; + const uint32_t Bc = fa.first.Bc; + const bool aligned = fa.first.aligned; + const bool f32acc = fa.first.f32acc; + const uint32_t fa_sgs = fa.first.subgroup_size; + const bool fa_ds = fa.first.subgroup_size == 0; - if (size == 0) { - buf->size = 0; - return buf; - } + const bool bf16_kv = fa.first.k_type == GGML_TYPE_BF16; - vk::BufferUsageFlags usage_flags = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst; - vk::MemoryAllocateFlags mem_flags {}; - if (device->buffer_device_address) { - usage_flags |= vk::BufferUsageFlagBits::eShaderDeviceAddress; - mem_flags |= vk::MemoryAllocateFlagBits::eDeviceAddress; + const void * spv_data; + size_t spv_size; + const char *name; + if (bf16_kv) { +#if defined(VK_KHR_shader_bfloat16) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + if (!device->coopmat_bf16_support) continue; + spv_data = flash_attn_f32_f16_bf16_cm1_data; + spv_size = flash_attn_f32_f16_bf16_cm1_len; + name = aligned ? "flash_attn_f32_bf16_aligned_cm1" : "flash_attn_f32_bf16_cm1"; +#else + continue; +#endif + } else { + if (f32acc) { spv_data = flash_attn_f32_f16_cm1_data; spv_size = flash_attn_f32_f16_cm1_len; } + else { spv_data = flash_attn_f32_f16_f16acc_cm1_data; spv_size = flash_attn_f32_f16_f16acc_cm1_len; } + name = aligned ? "flash_attn_f32_f16_aligned_cm1" : "flash_attn_f32_f16_cm1"; + } + ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 8, + sizeof(vk_flash_attn_push_constants), {Br, 1, 1}, + get_fa_spec_constants(fa.first), aligned ? Bc : 1, true, + !fa_ds, !fa_ds ? fa_sgs : 0); + } } +#endif - vk::BufferCreateInfo buffer_create_info{ - vk::BufferCreateFlags(), - size, - usage_flags, - vk::SharingMode::eExclusive, - 0, - nullptr, - }; +#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) + if (device->coopmat2) { + for (auto &fa : device->pipeline_flash_attn_f32_f16) { + if (fa.first.path != FA_COOPMAT2) continue; + const uint32_t Br = fa.first.Br; + const uint32_t Bc = fa.first.Bc; + const bool aligned = fa.first.aligned; + const bool f32acc = fa.first.f32acc; - vk::ExternalMemoryBufferCreateInfo external_memory_bci; - if (import_ptr) { - external_memory_bci.handleTypes = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT; - buffer_create_info.setPNext(&external_memory_bci); + const bool bf16_kv = fa.first.k_type == GGML_TYPE_BF16; + const void * spv_data; + size_t spv_size; + const char * name; + if (bf16_kv) { +#if defined(VK_KHR_shader_bfloat16) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + if (!device->coopmat2_bf16_support) continue; + spv_data = flash_attn_f32_f16_bf16_cm2_data; + spv_size = flash_attn_f32_f16_bf16_cm2_len; + name = aligned ? "flash_attn_f32_bf16_aligned_cm2" : "flash_attn_f32_bf16_cm2"; +#else + continue; +#endif + } else if (aligned) { + if (f32acc) { spv_data = flash_attn_f32_f16_cm2_data; spv_size = flash_attn_f32_f16_cm2_len; name = "flash_attn_f32_f16_aligned_f32acc_cm2"; } + else { spv_data = flash_attn_f32_f16_f16acc_cm2_data; spv_size = flash_attn_f32_f16_f16acc_cm2_len; name = "flash_attn_f32_f16_aligned_f16acc_cm2"; } + } else { + if (f32acc) { spv_data = flash_attn_f32_f16_cm2_data; spv_size = flash_attn_f32_f16_cm2_len; name = "flash_attn_f32_f16_f32acc_cm2"; } + else { spv_data = flash_attn_f32_f16_f16acc_cm2_data; spv_size = flash_attn_f32_f16_f16acc_cm2_len; name = "flash_attn_f32_f16_f16acc_cm2"; } + } + ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 8, + sizeof(vk_flash_attn_push_constants), {Br, 1, 1}, + get_fa_spec_constants(fa.first), aligned ? Bc : 1, true, false, 0); + } } +#endif - buf->buffer = device->device.createBuffer(buffer_create_info); - - vk::MemoryRequirements mem_req = device->device.getBufferMemoryRequirements(buf->buffer); + auto const &ggml_vk_mul_mm_spec = [&device](std::vector<uint32_t> spec, bool aligned) { + spec.push_back(aligned ? 1u : 0u); // constantID=11: ALIGNED + if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && + device->driver_id == vk::DriverId::eIntelProprietaryWindows) { + spec.push_back(0u); // constantID=12: SHMEM_STRIDE_PAD = 0 + spec.push_back(1u); // constantID=13: APPLY_SLM_A_RESHAPE = true + } + return spec; + }; - vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties(); + auto const &ggml_vk_mul_mm_spec_quant = [&device](std::vector<uint32_t> spec, bool aligned, uint32_t type) { + spec.push_back(aligned ? 1u : 0u); // constantID=11: ALIGNED + spec.push_back(type); // constantID=12: MmTypeA + if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && + device->driver_id == vk::DriverId::eIntelProprietaryWindows) { + spec.push_back(0u); // constantID=13: SHMEM_STRIDE_PAD = 0 + spec.push_back(1u); // constantID=14: APPLY_SLM_A_RESHAPE = true + } + return spec; + }; - const vk::MemoryPriorityAllocateInfoEXT mem_priority_info { 1.0f }; + static const ggml_type non_lut_quant_types[] = { + GGML_TYPE_Q1_0, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, + GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_TQ1_0, GGML_TYPE_TQ2_0, + }; - vk::MemoryAllocateFlagsInfo mem_flags_info { mem_flags }; +#define FOR_EACH_LUT_TYPE_NONFP4(X) \ + X(GGML_TYPE_IQ1_S, iq1_s) \ + X(GGML_TYPE_IQ1_M, iq1_m) \ + X(GGML_TYPE_IQ2_XXS, iq2_xxs) \ + X(GGML_TYPE_IQ2_XS, iq2_xs) \ + X(GGML_TYPE_IQ2_S, iq2_s) \ + X(GGML_TYPE_IQ3_XXS, iq3_xxs) \ + X(GGML_TYPE_IQ3_S, iq3_s) \ + X(GGML_TYPE_IQ4_XS, iq4_xs) \ + X(GGML_TYPE_IQ4_NL, iq4_nl) +#define FOR_EACH_LUT_FP4_TYPE(X) \ + X(GGML_TYPE_MXFP4, mxfp4) \ + X(GGML_TYPE_NVFP4, nvfp4) +#define FOR_EACH_LUT_TYPE(X) \ + FOR_EACH_LUT_TYPE_NONFP4(X) \ + FOR_EACH_LUT_FP4_TYPE(X) - if (device->memory_priority) { - mem_flags_info.setPNext(&mem_priority_info); - } + // 5 stock + 1 for the weight-paging per-expert offset array (binding 5). + const int mul_mat_id_param_count = 6; - if (import_ptr) { - vk::MemoryHostPointerPropertiesEXT host_pointer_props; - try { - host_pointer_props = device->device.getMemoryHostPointerPropertiesEXT(vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT, import_ptr); - } catch (vk::SystemError& e) { - GGML_LOG_WARN("ggml_vulkan: Failed getMemoryHostPointerPropertiesEXT (%s)\n", e.what()); - device->device.destroyBuffer(buf->buffer); - return {}; - } - vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties(); - - uint32_t memory_type_idx; - vk::MemoryPropertyFlags property_flags = *req_flags_list.begin(); - for (memory_type_idx = 0; memory_type_idx < 32; ++memory_type_idx) { - if (!(host_pointer_props.memoryTypeBits & (1u << memory_type_idx))) { - continue; - } - if (!(mem_req.memoryTypeBits & (1u << memory_type_idx))) { - continue; + using spec_fn_t = std::function<std::vector<uint32_t>(const std::vector<uint32_t>&, bool)>; + auto const &create_mm_pipelines = [&]( + const vk_matmul_pipeline_key& key, + const std::vector<vk_tile_config>& tile_configs, + const std::string& shader_name, size_t spv_len, const void* spv_data, + uint32_t push_constant_size, uint32_t param_count, + const spec_fn_t& spec_fn, + bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0, + bool create_aligned = true, bool pin_subgroup_to_warp = false + ) { + auto& vec = device->pipeline_matmul[key]; + const bool first_call = vec.empty(); + for (size_t i = 0; i < tile_configs.size(); i++) { + const auto& tc = tile_configs[i]; + + // Intel coopmat1 pins the required subgroup size to each warptile's WARP element. + const uint32_t rsgs = pin_subgroup_to_warp ? tc.warptile[WARP_SIZE_IDX] : required_subgroup_size; + const bool rfs = require_full_subgroups || pin_subgroup_to_warp; + + if (first_call) { + vk_matmul_pipeline_pair pair{}; + pair.align = tc.align; + std::string suffix = "_" + std::to_string(i); + pair.unaligned = std::make_shared<vk_pipeline_struct>(); + if (create_aligned) { + pair.aligned = std::make_shared<vk_pipeline_struct>(); + } + vec.push_back(pair); } - vk::MemoryType memory_type = mem_props.memoryTypes[memory_type_idx]; - // check for visible+coherent+cached. Other flags (e.g. devicelocal) are allowed - if ((memory_type.propertyFlags & property_flags) == property_flags) { - property_flags = memory_type.propertyFlags; - break; + ggml_vk_create_pipeline(device, vec[i].unaligned, + vec[i].unaligned->name.empty() ? (shader_name + "_" + std::to_string(i)).c_str() : vec[i].unaligned->name.c_str(), + spv_len, spv_data, "main", param_count, push_constant_size, + tc.wg_denoms, spec_fn(tc.warptile, false), 1, + disable_robustness, rfs, rsgs); + + if (vec[i].aligned) { + ggml_vk_create_pipeline(device, vec[i].aligned, + vec[i].aligned->name.empty() ? (shader_name + "_aligned_" + std::to_string(i)).c_str() : vec[i].aligned->name.c_str(), + spv_len, spv_data, "main", param_count, push_constant_size, + tc.wg_denoms, spec_fn(tc.warptile, true), tc.align, + disable_robustness, rfs, rsgs); } } - if (memory_type_idx == 32) { - GGML_LOG_WARN("ggml_vulkan: Memory type for host allocation not found\n"); - device->device.destroyBuffer(buf->buffer); - return {}; - } + }; - buf->memory_property_flags = mem_props.memoryTypes[memory_type_idx].propertyFlags; - try { - vk::ImportMemoryHostPointerInfoEXT import_info; - import_info.handleType = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT; - import_info.pHostPointer = import_ptr; - import_info.setPNext(&mem_flags_info); - buf->device_memory = device->device.allocateMemory({ size, memory_type_idx, &import_info }); - } catch (const vk::SystemError& e) { + auto filter_tc = [&](const std::vector<vk_tile_config>& configs, ggml_type type, bool is_id, bool is_int = false) -> std::vector<vk_tile_config> { + std::vector<vk_tile_config> result; + bool enabled[3]; + if (is_int) { + enabled[0] = is_id ? device->mul_mat_id_s_int[type] : device->mul_mat_s_int[type]; + enabled[1] = is_id ? device->mul_mat_id_m_int[type] : device->mul_mat_m_int[type]; + enabled[2] = is_id ? device->mul_mat_id_l_int[type] : device->mul_mat_l_int[type]; + } else { + enabled[0] = is_id ? device->mul_mat_id_s[type] : device->mul_mat_s[type]; + enabled[1] = is_id ? device->mul_mat_id_m[type] : device->mul_mat_m[type]; + enabled[2] = is_id ? device->mul_mat_id_l[type] : device->mul_mat_l[type]; } - } else { - for (auto it = req_flags_list.begin(); it != req_flags_list.end(); it++) { - const auto & req_flags = *it; + for (size_t i = 0; i < configs.size() && i < 3; i++) { + if (enabled[i]) result.push_back(configs[i]); + } + return result; + }; - const std::vector<uint32_t> memory_type_indices = ggml_vk_find_memory_properties(&mem_props, &mem_req, req_flags); + std::vector<vk_tile_config> tc_mm = {{s_warptile, s_wg_denoms, s_align}, {m_warptile, m_wg_denoms, m_align}, {l_warptile, l_wg_denoms, l_align}}; + std::vector<vk_tile_config> tc_mmq = {{s_warptile_mmq, s_mmq_wg_denoms, s_align}, {m_warptile_mmq, m_mmq_wg_denoms, m_align}, {l_warptile_mmq, l_mmq_wg_denoms, l_align}}; - if (memory_type_indices.empty()) { - continue; +#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) + if (device->coopmat2) { + auto const &ggml_vk_mul_mm_cm2_spec = [&](std::vector<uint32_t> spec, bool aligned, uint32_t type = UINT32_MAX) { + spec.push_back(aligned ? 1u : 0u); // ALIGNED + spec.push_back(device->subgroup_size); // subgroup_size + if (type != UINT32_MAX) { + spec.push_back(type); // MmTypeA + spec.push_back((uint32_t)ggml_type_size((ggml_type)type)); // MmABlockBytes } + return spec; + }; - bool done = false; + std::vector<vk_tile_config> tc_mmq_k = {{s_warptile_mmq_k, s_mmq_wg_denoms_k, s_align}, {m_warptile_mmq_k, m_mmq_wg_denoms_k, m_align}, {l_warptile_mmq_k, l_mmq_wg_denoms_k, l_align}}; + std::vector<vk_tile_config> tc_mmqid = {{s_warptile_mmqid, s_mmqid_wg_denoms, s_align}, {m_warptile_mmqid, m_mmqid_wg_denoms, m_align}, {l_warptile_mmqid, l_mmqid_wg_denoms, l_align}}; - for (auto mtype_it = memory_type_indices.begin(); mtype_it != memory_type_indices.end(); mtype_it++) { - try { - buf->device_memory = device->device.allocateMemory({ mem_req.size, *mtype_it, &mem_flags_info }); - buf->memory_property_flags = mem_props.memoryTypes[*mtype_it].propertyFlags; - done = true; - break; - } catch (const vk::SystemError& e) { - // loop and retry - // during last attempt throw the exception - if (it + 1 == req_flags_list.end() && mtype_it + 1 == memory_type_indices.end()) { - device->device.destroyBuffer(buf->buffer); - throw e; - } - } - } + spec_fn_t cm2_spec = [&](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a); }; - if (done) { - break; + // F16 x F16 + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm2_len, matmul_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm2_len, matmul_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); +#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + if (device->coopmat_bf16_support) { + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm2_len, matmul_bf16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + } +#endif + for (const auto type : non_lut_quant_types) { + // regression in unified shader on Ampere + if (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K) { + continue; } + auto& tc = ((type >= GGML_TYPE_Q2_K && type <= GGML_TYPE_Q6_K) || type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0) ? tc_mmq_k : tc_mmq; + spec_fn_t qs = [&, type](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); }; + create_mm_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); + create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16", matmul_quant_f16_cm2_len, matmul_quant_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); + } + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q4_k_f16_f16acc", matmul_q4_k_f16_f16acc_cm2_len, matmul_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q4_k_f16", matmul_q4_k_f16_cm2_len, matmul_q4_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q5_k_f16_f16acc", matmul_q5_k_f16_f16acc_cm2_len, matmul_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q5_k_f16", matmul_q5_k_f16_cm2_len, matmul_q5_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); +#define X_CM2(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmq, TYPE, false); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm2_len, matmul_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm2_len, matmul_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + } } + FOR_EACH_LUT_TYPE_NONFP4(X_CM2) +#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) + if (device->ocp_fp4) { +#define X_CM2_OCP(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmq, TYPE, false); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm2_len, matmul_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16_ocp", matmul_##tstr##_f16_ocp_cm2_len, matmul_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + } } + FOR_EACH_LUT_FP4_TYPE(X_CM2_OCP) +#undef X_CM2_OCP + } else +#endif + { + FOR_EACH_LUT_FP4_TYPE(X_CM2) } - } +#undef X_CM2 - if (!buf->device_memory) { - device->device.destroyBuffer(buf->buffer); - throw vk::OutOfDeviceMemoryError("No suitable memory type found"); - } + GGML_ASSERT(device->subgroup_ballot); + + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_subgroup_f16_f16acc", matmul_id_subgroup_f16_f16acc_cm2_len, matmul_id_subgroup_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_cm2_len, matmul_id_subgroup_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); +#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + if (device->coopmat_bf16_support) { + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm2_len, matmul_id_subgroup_bf16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + } +#endif + for (const auto type : non_lut_quant_types) { + if (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K) { + continue; + } + spec_fn_t qs_id = [&, type](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); }; + create_mm_pipelines({type, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f16_f16acc", matmul_id_subgroup_quant_f16_f16acc_cm2_len, matmul_id_subgroup_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); + create_mm_pipelines({type, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm2_len, matmul_id_subgroup_quant_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); + } + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_q4_k_f16_f16acc", matmul_id_subgroup_q4_k_f16_f16acc_cm2_len, matmul_id_subgroup_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_q4_k_f16", matmul_id_subgroup_q4_k_f16_cm2_len, matmul_id_subgroup_q4_k_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_q5_k_f16_f16acc", matmul_id_subgroup_q5_k_f16_f16acc_cm2_len, matmul_id_subgroup_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_q5_k_f16", matmul_id_subgroup_q5_k_f16_cm2_len, matmul_id_subgroup_q5_k_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); +#define X_CM2_ID(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmqid, TYPE, true); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, true}, tc, "matmul_id_subgroup_" #tstr "_f16_f16acc", matmul_id_subgroup_##tstr##_f16_f16acc_cm2_len, matmul_id_subgroup_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, false}, tc, "matmul_id_subgroup_" #tstr "_f16", matmul_id_subgroup_##tstr##_f16_cm2_len, matmul_id_subgroup_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + } } + FOR_EACH_LUT_TYPE_NONFP4(X_CM2_ID) +#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) + if (device->ocp_fp4) { +#define X_CM2_ID_OCP(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmqid, TYPE, true); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, true}, tc, "matmul_id_subgroup_" #tstr "_f16_ocp_f16acc", matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm2_len, matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, false}, tc, "matmul_id_subgroup_" #tstr "_f16_ocp", matmul_id_subgroup_##tstr##_f16_ocp_cm2_len, matmul_id_subgroup_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + } } + FOR_EACH_LUT_FP4_TYPE(X_CM2_ID_OCP) +#undef X_CM2_ID_OCP + } else +#endif + { + FOR_EACH_LUT_FP4_TYPE(X_CM2_ID) + } +#undef X_CM2_ID + } else +#endif // defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) +#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + if (device->coopmat_support) { + spec_fn_t cm1_spec = [&](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); }; - buf->ptr = nullptr; + // Intel coopmat1 pins each pipeline's required subgroup size to its warptile WARP element. + const bool cm1_pin = device->vendor_id == VK_VENDOR_ID_INTEL; - if (import_ptr) { - buf->ptr = import_ptr; - } else { - if (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { - buf->ptr = device->device.mapMemory(buf->device_memory, 0, VK_WHOLE_SIZE); + // Intel coopmat1 uses a dedicated large-tile config for quant matmul_id. + std::vector<vk_tile_config> tc_mmq_id = tc_mmq; + if (cm1_pin) { + tc_mmq_id[2] = { { 512, 128, 128, 32, 32, 32, 2, device->coopmat_m, device->coopmat_n, device->coopmat_k, 32 }, { 128, 128, 1 }, 32 }; } - } - device->device.bindBufferMemory(buf->buffer, buf->device_memory, 0); + auto cm1_create = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, cm1_spec, false, true, 0, true, cm1_pin); + }; + auto cm1_create_quant = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc) { + spec_fn_t qs = [&, type_a=key.type_a](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_spec_quant(wt, a, (uint32_t)type_a); }; + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, qs, false, true, 0, true, cm1_pin); + }; - buf->device = device; - buf->size = size; + cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", matmul_f32_f32_cm1_len, matmul_f32_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", matmul_f32_f16_cm1_len, matmul_f32_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + if (device->coopmat_acc_f16_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm1_len, matmul_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, false, true}, tc_mm, "matmul_f16_f32_f16acc", matmul_f16_f32_f16acc_cm1_len, matmul_f16_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + if (device->coopmat_acc_f32_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm1_len, matmul_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", matmul_f16_f32_cm1_len, matmul_f16_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } +#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + if (device->coopmat_bf16_support) { + cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm1_len, matmul_bf16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } +#endif + for (const auto type : non_lut_quant_types) { + if (device->coopmat_acc_f16_support) { + cm1_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_f16acc", matmul_quant_f32_f16acc_cm1_len, matmul_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create_quant({type, GGML_TYPE_F16, false, true}, tc_mmq, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm1_len, matmul_quant_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + if (device->coopmat_acc_f32_support) { + cm1_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_cm1_len, matmul_quant_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create_quant({type, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_quant_f16", matmul_quant_f16_cm1_len, matmul_quant_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + } + // The _f16 variants provide the f16 B-type pipeline used when y_non_contig converts f32->f16. +#define X_CM1(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_f16acc", matmul_##tstr##_f32_f16acc_cm1_len, matmul_##tstr##_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + cm1_create({TYPE, GGML_TYPE_F16, false, true}, tc_mmq, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm1_len, matmul_##tstr##_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", matmul_##tstr##_f32_cm1_len, matmul_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + cm1_create({TYPE, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm1_len, matmul_##tstr##_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + } + FOR_EACH_LUT_TYPE_NONFP4(X_CM1) +#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) + if (device->ocp_fp4) { +#define X_CM1_OCP(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_ocp_f16acc", matmul_##tstr##_f32_ocp_f16acc_cm1_len, matmul_##tstr##_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + cm1_create({TYPE, GGML_TYPE_F16, false, true}, tc_mmq, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm1_len, matmul_##tstr##_f16_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32_ocp", matmul_##tstr##_f32_ocp_cm1_len, matmul_##tstr##_f32_ocp_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + cm1_create({TYPE, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_" #tstr "_f16_ocp", matmul_##tstr##_f16_ocp_cm1_len, matmul_##tstr##_f16_ocp_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + } + FOR_EACH_LUT_FP4_TYPE(X_CM1_OCP) +#undef X_CM1_OCP + } else +#endif + { + FOR_EACH_LUT_FP4_TYPE(X_CM1) + } +#undef X_CM1 - if (device->buffer_device_address) { - const vk::BufferDeviceAddressInfo addressInfo(buf->buffer); - buf->bda_addr = device->device.getBufferAddress(addressInfo); - } + GGML_ASSERT(device->subgroup_ballot); - device->memory_logger->log_allocation(buf, size); + cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_subgroup_f32_f32", matmul_id_subgroup_f32_f32_cm1_len, matmul_id_subgroup_f32_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + if (device->coopmat_acc_f16_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_subgroup_f16_f16acc", matmul_id_subgroup_f16_f16acc_cm1_len, matmul_id_subgroup_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_mm, "matmul_id_subgroup_f16_f32_f16acc", matmul_id_subgroup_f16_f32_f16acc_cm1_len, matmul_id_subgroup_f16_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + if (device->coopmat_acc_f32_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_cm1_len, matmul_id_subgroup_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_cm1_len, matmul_id_subgroup_f16_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } +#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + if (device->coopmat_bf16_support) { + cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm1_len, matmul_id_subgroup_bf16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } +#endif + for (const auto type : non_lut_quant_types) { + if (device->coopmat_acc_f16_support) { + cm1_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmq_id, "matmul_id_subgroup_quant_f32_f16acc", matmul_id_subgroup_quant_f32_f16acc_cm1_len, matmul_id_subgroup_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create_quant({type, GGML_TYPE_F16, true, true}, tc_mmq_id, "matmul_id_subgroup_quant_f16_f16acc", matmul_id_subgroup_quant_f16_f16acc_cm1_len, matmul_id_subgroup_quant_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + if (device->coopmat_acc_f32_support) { + cm1_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_cm1_len, matmul_id_subgroup_quant_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create_quant({type, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm1_len, matmul_id_subgroup_quant_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + } + // The _f16 variants provide the f16 B-type pipeline used when y_non_contig converts f32->f16. +#define X_CM1_ID(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32_f16acc", matmul_id_subgroup_##tstr##_f32_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + cm1_create({TYPE, GGML_TYPE_F16, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16_f16acc", matmul_id_subgroup_##tstr##_f16_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32", matmul_id_subgroup_##tstr##_f32_cm1_len, matmul_id_subgroup_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + cm1_create({TYPE, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16", matmul_id_subgroup_##tstr##_f16_cm1_len, matmul_id_subgroup_##tstr##_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + } + FOR_EACH_LUT_TYPE_NONFP4(X_CM1_ID) +#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) + if (device->ocp_fp4) { +#define X_CM1_ID_OCP(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32_ocp_f16acc", matmul_id_subgroup_##tstr##_f32_ocp_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + cm1_create({TYPE, GGML_TYPE_F16, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16_ocp_f16acc", matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32_ocp", matmul_id_subgroup_##tstr##_f32_ocp_cm1_len, matmul_id_subgroup_##tstr##_f32_ocp_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + cm1_create({TYPE, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16_ocp", matmul_id_subgroup_##tstr##_f16_ocp_cm1_len, matmul_id_subgroup_##tstr##_f16_ocp_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + } + FOR_EACH_LUT_FP4_TYPE(X_CM1_ID_OCP) +#undef X_CM1_ID_OCP + } else +#endif + { + FOR_EACH_LUT_FP4_TYPE(X_CM1_ID) + } +#undef X_CM1_ID + } else +#endif // defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + { + // Helper for subgroup path with dot2 selection and filtering + auto sg_create = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, + uint32_t rsgs = 0) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, + [&](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); }, + false, rsgs > 0, rsgs); + }; + auto sg_create_quant = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, + uint32_t rsgs = 0) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) { + spec_fn_t qs = [&, type_a=key.type_a](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_spec_quant(wt, a, (uint32_t)type_a); }; + create_mm_pipelines(key, tc, name, len, data, pc_size, pc, qs, false, rsgs > 0, rsgs); + } + }; +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + auto sg_create_mmq = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, + uint32_t rsgs = 0) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id, true); + if (!tc.empty()) { + spec_fn_t identity = [](const std::vector<uint32_t>& wt, bool) { return wt; }; + create_mm_pipelines(key, tc, name, len, data, pc_size, pc, identity, false, rsgs > 0, rsgs, false); + } + }; +#endif - return buf; -} + std::vector<vk_tile_config> tc_id = {{s_warptile_id, s_wg_denoms, s_align}, {m_warptile_id, m_wg_denoms, m_align}, {l_warptile_id, l_wg_denoms, l_align}}; + std::vector<vk_tile_config> tc_mmqid = {{s_warptile_mmqid, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid, l_mmq_wg_denoms, l_align}}; -static vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0)) { - try { - return ggml_vk_create_buffer(device, size, {req_flags, fallback_flags}); - } catch (const vk::SystemError& e) { - std::cerr << "ggml_vulkan: Memory allocation of size " << size << " failed." << std::endl; - std::cerr << "ggml_vulkan: " << e.what() << std::endl; - throw e; - } -} + if (device->fp16) { + // FP16 subgroup path - with dot2 runtime selection + #define SPV_DOT2(NAME) (device->dot2_f16 ? NAME ## _dot2_len : NAME ## _len), (device->dot2_f16 ? NAME ## _dot2_data : NAME ## _data) + #define SPV_DOT2_F16ACC(NAME) (device->dot2_f16 ? NAME ## _dot2_f16acc_len : NAME ## _f16acc_len), (device->dot2_f16 ? NAME ## _dot2_f16acc_data : NAME ## _f16acc_data) + + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", SPV_DOT2(matmul_f32_f32), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", SPV_DOT2(matmul_f32_f16), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", SPV_DOT2_F16ACC(matmul_f16), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", SPV_DOT2(matmul_f16), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, true}, tc_mm, "matmul_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_f16_f32), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", SPV_DOT2(matmul_f16_f32), sizeof(vk_mat_mat_push_constants), 3); + // BF16 - no dot2 + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_len, matmul_bf16_data, sizeof(vk_mat_mat_push_constants), 3); + + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_quant_f32), sizeof(vk_mat_mat_push_constants), 3); + sg_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", SPV_DOT2(matmul_quant_f32), sizeof(vk_mat_mat_push_constants), 3); + } + #define X_SG(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_##tstr##_f32), sizeof(vk_mat_mat_push_constants), 3); \ + sg_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", SPV_DOT2(matmul_##tstr##_f32), sizeof(vk_mat_mat_push_constants), 3); + FOR_EACH_LUT_TYPE(X_SG) +#undef X_SG -static vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size) { - vk_buffer buf; - // GGML_VK_HOST_VISIBLE_VIDMEM_MAX_BYTES: keep the host-visible-device-local - // preference (the BAR) for allocations up to N bytes, and force larger ones - // straight to plain device-local. - // - // WHY. On a card without ReBAR the host-visible device-local heap IS the - // small BAR window (256 MB on an RX 480). AMD does not fail an - // oversubscribed allocation there -- it migrates to GTT -- so bulk weight - // buffers past the window silently land in system RAM and every matmul - // against them streams over PCIe (measured 2026-08-01: 190 us -> 1413 us for - // one expert group). GGML_VK_DISABLE_HOST_VISIBLE_VIDMEM avoids that, but it - // is all-or-nothing: it also pushes SMALL, frequently-rewritten buffers out - // of the BAR, and writing those then costs a staging copy + submit + fence - // instead of a memcpy (measured: a 16 KB per-request activation upload went - // 0.05 ms -> 1.69 ms, +2.44 s over a 64-token run). - // A size split gets both: small buffers keep memcpy writes, bulk weights keep - // real VRAM bandwidth. Unset (0) preserves the previous behaviour exactly. - static const size_t host_visible_max = [] { - const char * e = getenv("GGML_VK_HOST_VISIBLE_VIDMEM_MAX_BYTES"); - return (e != nullptr && e[0] != '\0') ? (size_t) strtoull(e, nullptr, 10) : (size_t) 0; - }(); - const bool force_device_local = host_visible_max != 0 && size > host_visible_max; - // GGML_VK_ALLOC_LOG=1: report every device buffer and whether it ended up - // host-visible (memcpy writes) or device-local-only (staging + submit + - // fence per write, ggml_vk_buffer_write_2d). - const bool alloc_log = getenv("GGML_VK_ALLOC_LOG") != nullptr; - try { - if (force_device_local) { - // Large: never take the BAR. Mirrors the disable_host_visible_vidmem - // branch below, including its sysmem-fallback behaviour. - if (device->allow_sysmem_fallback) { - buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal, - vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); - } else { - buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal}); +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (device->integer_dot_product) { + std::vector<vk_tile_config> tc_mmq_int = {{s_warptile_mmq_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int, l_mmq_wg_denoms, l_align}}; + std::vector<vk_tile_config> tc_mmq_int_k = {{s_warptile_mmq_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q2_0_q8_1", matmul_q2_0_q8_1_len, matmul_q2_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_0_q8_1", matmul_q4_0_q8_1_len, matmul_q4_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_1_q8_1", matmul_q4_1_q8_1_len, matmul_q4_1_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_0_q8_1", matmul_q5_0_q8_1_len, matmul_q5_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_1_q8_1", matmul_q5_1_q8_1_len, matmul_q5_1_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q8_0_q8_1", matmul_q8_0_q8_1_len, matmul_q8_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_mxfp4_q8_1", matmul_mxfp4_q8_1_len, matmul_mxfp4_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_IQ4_XS, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_iq4_xs_q8_1", matmul_iq4_xs_q8_1_len, matmul_iq4_xs_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q2_k_q8_1", matmul_q2_k_q8_1_len, matmul_q2_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q3_k_q8_1", matmul_q3_k_q8_1_len, matmul_q3_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q4_k_q8_1", matmul_q4_k_q8_1_len, matmul_q4_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q5_k_q8_1", matmul_q5_k_q8_1_len, matmul_q5_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q6_k_q8_1", matmul_q6_k_q8_1_len, matmul_q6_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_IQ3_S, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_iq3_s_q8_1", matmul_iq3_s_q8_1_len, matmul_iq3_s_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); } - } else if (device->prefer_host_memory) { - buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, - vk::MemoryPropertyFlagBits::eDeviceLocal}); - } else if (device->uma) { - // On UMA, prefer host-visible memory so direct tensor borrowing works. - // If unavailable, fall back to device-local memory. - buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, - vk::MemoryPropertyFlagBits::eDeviceLocal, - vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); - } else if (device->disable_host_visible_vidmem) { - if (device->allow_sysmem_fallback) { - buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal, - vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); +#endif + + if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f32_f32", SPV_DOT2(matmul_id_subgroup_f32_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_id, "matmul_id_subgroup_f16_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_id, "matmul_id_subgroup_f16", SPV_DOT2(matmul_id_subgroup_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_id, "matmul_id_subgroup_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", SPV_DOT2(matmul_id_subgroup_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + // BF16 id - no dot2 + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_id, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_len, matmul_id_subgroup_bf16_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f32", SPV_DOT2(matmul_id_subgroup_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + } + #define X_SG_ID_SUB(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32", SPV_DOT2(matmul_id_subgroup_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + FOR_EACH_LUT_TYPE(X_SG_ID_SUB) +#undef X_SG_ID_SUB +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (device->integer_dot_product) { + std::vector<vk_tile_config> tc_mmqid_int = {{s_warptile_mmqid_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int, l_mmq_wg_denoms, l_align}}; + std::vector<vk_tile_config> tc_mmqid_int_k = {{s_warptile_mmqid_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q2_0_q8_1", matmul_id_subgroup_q2_0_q8_1_len, matmul_id_subgroup_q2_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q4_0_q8_1", matmul_id_subgroup_q4_0_q8_1_len, matmul_id_subgroup_q4_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q4_1_q8_1", matmul_id_subgroup_q4_1_q8_1_len, matmul_id_subgroup_q4_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q5_0_q8_1", matmul_id_subgroup_q5_0_q8_1_len, matmul_id_subgroup_q5_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q5_1_q8_1", matmul_id_subgroup_q5_1_q8_1_len, matmul_id_subgroup_q5_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q8_0_q8_1", matmul_id_subgroup_q8_0_q8_1_len, matmul_id_subgroup_q8_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_mxfp4_q8_1", matmul_id_subgroup_mxfp4_q8_1_len, matmul_id_subgroup_mxfp4_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_IQ4_XS, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_iq4_xs_q8_1", matmul_id_subgroup_iq4_xs_q8_1_len, matmul_id_subgroup_iq4_xs_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q2_k_q8_1", matmul_id_subgroup_q2_k_q8_1_len, matmul_id_subgroup_q2_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q3_k_q8_1", matmul_id_subgroup_q3_k_q8_1_len, matmul_id_subgroup_q3_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q4_k_q8_1", matmul_id_subgroup_q4_k_q8_1_len, matmul_id_subgroup_q4_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q5_k_q8_1", matmul_id_subgroup_q5_k_q8_1_len, matmul_id_subgroup_q5_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q6_k_q8_1", matmul_id_subgroup_q6_k_q8_1_len, matmul_id_subgroup_q6_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_IQ3_S, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_iq3_s_q8_1", matmul_id_subgroup_iq3_s_q8_1_len, matmul_id_subgroup_iq3_s_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + } +#endif } else { - buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal}); + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f32_f32", SPV_DOT2(matmul_id_f32_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_f16_f16acc", SPV_DOT2_F16ACC(matmul_id_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_f16", SPV_DOT2(matmul_id_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_mm, "matmul_id_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f16_f32", SPV_DOT2(matmul_id_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + // BF16 id - no dot2 + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_bf16", matmul_id_bf16_len, matmul_id_bf16_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_quant_f32", SPV_DOT2(matmul_id_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + #define X_SG_ID(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_" #tstr "_f32", SPV_DOT2(matmul_id_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + FOR_EACH_LUT_TYPE(X_SG_ID) +#undef X_SG_ID +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (device->integer_dot_product) { + std::vector<vk_tile_config> tc_mmqid_int = {{s_warptile_mmqid_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int, l_mmq_wg_denoms, l_align}}; + std::vector<vk_tile_config> tc_mmqid_int_k = {{s_warptile_mmqid_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q2_0_q8_1", matmul_id_q2_0_q8_1_len, matmul_id_q2_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q4_0_q8_1", matmul_id_q4_0_q8_1_len, matmul_id_q4_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q4_1_q8_1", matmul_id_q4_1_q8_1_len, matmul_id_q4_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q5_0_q8_1", matmul_id_q5_0_q8_1_len, matmul_id_q5_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q5_1_q8_1", matmul_id_q5_1_q8_1_len, matmul_id_q5_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q8_0_q8_1", matmul_id_q8_0_q8_1_len, matmul_id_q8_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_mxfp4_q8_1", matmul_id_mxfp4_q8_1_len, matmul_id_mxfp4_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_IQ4_XS, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_iq4_xs_q8_1", matmul_id_iq4_xs_q8_1_len, matmul_id_iq4_xs_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q2_k_q8_1", matmul_id_q2_k_q8_1_len, matmul_id_q2_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q3_k_q8_1", matmul_id_q3_k_q8_1_len, matmul_id_q3_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q4_k_q8_1", matmul_id_q4_k_q8_1_len, matmul_id_q4_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q5_k_q8_1", matmul_id_q5_k_q8_1_len, matmul_id_q5_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q6_k_q8_1", matmul_id_q6_k_q8_1_len, matmul_id_q6_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_IQ3_S, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_iq3_s_q8_1", matmul_id_iq3_s_q8_1_len, matmul_id_iq3_s_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } +#endif } + #undef SPV_DOT2 + #undef SPV_DOT2_F16ACC } else { - // use rebar if available, otherwise fallback to device only visible memory - if (device->allow_sysmem_fallback) { - buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, - vk::MemoryPropertyFlagBits::eDeviceLocal, - vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); + // FP32-only fallback path + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", matmul_f32_f32_fp32_len, matmul_f32_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", matmul_f32_f16_fp32_len, matmul_f32_f16_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_fp32_len, matmul_f16_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", matmul_f16_f32_fp32_len, matmul_f16_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_fp32_len, matmul_bf16_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_fp32_len, matmul_quant_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + } + #define X_SG_FP32(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", matmul_##tstr##_f32_fp32_len, matmul_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + FOR_EACH_LUT_TYPE(X_SG_FP32) +#undef X_SG_FP32 + +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (device->integer_dot_product) { + std::vector<vk_tile_config> tc_mmq_int = {{s_warptile_mmq_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int, l_mmq_wg_denoms, l_align}}; + std::vector<vk_tile_config> tc_mmq_int_k = {{s_warptile_mmq_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q2_0_q8_1", matmul_q2_0_q8_1_fp32_len, matmul_q2_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_0_q8_1", matmul_q4_0_q8_1_fp32_len, matmul_q4_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_1_q8_1", matmul_q4_1_q8_1_fp32_len, matmul_q4_1_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_0_q8_1", matmul_q5_0_q8_1_fp32_len, matmul_q5_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_1_q8_1", matmul_q5_1_q8_1_fp32_len, matmul_q5_1_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q8_0_q8_1", matmul_q8_0_q8_1_fp32_len, matmul_q8_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_IQ4_XS, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_iq4_xs_q8_1", matmul_iq4_xs_q8_1_fp32_len, matmul_iq4_xs_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q2_k_q8_1", matmul_q2_k_q8_1_fp32_len, matmul_q2_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q3_k_q8_1", matmul_q3_k_q8_1_fp32_len, matmul_q3_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q4_k_q8_1", matmul_q4_k_q8_1_fp32_len, matmul_q4_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q5_k_q8_1", matmul_q5_k_q8_1_fp32_len, matmul_q5_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q6_k_q8_1", matmul_q6_k_q8_1_fp32_len, matmul_q6_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_IQ3_S, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_iq3_s_q8_1", matmul_iq3_s_q8_1_fp32_len, matmul_iq3_s_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + } +#endif + + if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f32_f32", matmul_id_subgroup_f32_f32_fp32_len, matmul_id_subgroup_f32_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_id, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_fp32_len, matmul_id_subgroup_f16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_fp32_len, matmul_id_subgroup_f16_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_id, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_fp32_len, matmul_id_subgroup_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_fp32_len, matmul_id_subgroup_quant_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + } + #define X_SG_ID_SUB_FP32(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32", matmul_id_subgroup_##tstr##_f32_fp32_len, matmul_id_subgroup_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + FOR_EACH_LUT_TYPE(X_SG_ID_SUB_FP32) +#undef X_SG_ID_SUB_FP32 } else { - buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, - vk::MemoryPropertyFlagBits::eDeviceLocal}); + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f32_f32", matmul_id_f32_f32_fp32_len, matmul_id_f32_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_f16", matmul_id_f16_fp32_len, matmul_id_f16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f16_f32", matmul_id_f16_f32_fp32_len, matmul_id_f16_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_bf16", matmul_id_bf16_fp32_len, matmul_id_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_quant_f32", matmul_id_quant_f32_fp32_len, matmul_id_quant_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + #define X_SG_ID_FP32(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_" #tstr "_f32", matmul_id_##tstr##_f32_fp32_len, matmul_id_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + FOR_EACH_LUT_TYPE(X_SG_ID_FP32) +#undef X_SG_ID_FP32 } } - if (alloc_log && buf) { - const bool hv = (bool) (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible); - std::cerr << "ggml_vulkan: alloc " << size << " bytes -> " - << (hv ? "HOST_VISIBLE (memcpy writes)" : "device-local only (staged writes)") - << (force_device_local ? " [forced by size split]" : "") << std::endl; + } +#undef FOR_EACH_LUT_TYPE +#undef FOR_EACH_LUT_TYPE_NONFP4 +#undef FOR_EACH_LUT_FP4_TYPE + // BF16 fallback for coopmat devices without bf16 coopmat support + if ((device->coopmat2 || device->coopmat_support) +#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + && !device->coopmat_bf16_support +#endif + ) { + const uint32_t s_warptile_wm_bf16 = device->subgroup_size == 8 ? 8 : 32; + std::vector<vk_tile_config> tc_bf16_fb = { + {{ subgroup_size_32, 32, 32, 16, s_warptile_wm_bf16, 32, 2, 2, 2, 1, subgroup_size_8 }, {32, 32, 1}, s_align}, + {{ 128, 64, 64, 16, mm_warp_8, 32, 2, 4, 2, 1, mm_warp_8 }, {64, 64, 1}, m_align}, + {{ 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 }, {128, 128, 1}, l_align}, + }; + auto tc_bf16_filtered = filter_tc(tc_bf16_fb, GGML_TYPE_BF16, false); + auto tc_bf16_id_filtered = filter_tc(tc_bf16_fb, GGML_TYPE_BF16, true); + spec_fn_t bf16_spec = [&](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); }; + if (!tc_bf16_filtered.empty()) { + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_bf16_filtered, "matmul_bf16", matmul_bf16_fp32_len, matmul_bf16_fp32_data, sizeof(vk_mat_mat_push_constants), 3, bf16_spec); + } + if (!tc_bf16_id_filtered.empty()) { + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_bf16_id_filtered, "matmul_id_bf16", matmul_id_bf16_fp32_len, matmul_id_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, bf16_spec); } - } catch (const vk::SystemError& e) { - std::cerr << "ggml_vulkan: Device memory allocation of size " << size << " failed." << std::endl; - std::cerr << "ggml_vulkan: " << e.what() << std::endl; - throw e; } - return buf; -} - -static void ggml_vk_destroy_buffer(vk_buffer& buf) { - if (buf == nullptr) { - return; + // Set up tile selector functions + if (device->coopmat2) { + device->matmul_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t shader_core_count, + const std::vector<vk_matmul_pipeline_pair>& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + uint32_t last = (uint32_t)configs.size() - 1; + if (configs.size() == 2) { + uint32_t crossover = configs[0].unaligned->wg_denoms[1]; + return (n > crossover) ? 1 : 0; + } + // 3+ configs: s=0, m=1, l=2 + const uint32_t tiles_l = CEIL_DIV(m, configs[last].unaligned->wg_denoms[0]) * CEIL_DIV(n, configs[last].unaligned->wg_denoms[1]); + const uint32_t tiles_m = CEIL_DIV(m, configs[1].unaligned->wg_denoms[0]) * CEIL_DIV(n, configs[1].unaligned->wg_denoms[1]); + uint32_t crossover_large = configs[1].unaligned->wg_denoms[1]; + bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count || + (tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2); + if (n > crossover_large && prefer_large) return last; + uint32_t crossover_medium_m = configs[0].unaligned->wg_denoms[0]; + uint32_t crossover_medium_n = configs[0].unaligned->wg_denoms[1]; + if (m > crossover_medium_m && n > crossover_medium_n) return 1; + return 0; + }; + device->matmul_id_tile_selector = [](uint32_t /*m*/, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, + const std::vector<vk_matmul_pipeline_pair>& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + uint32_t last = (uint32_t)configs.size() - 1; + if (configs.size() == 2) { + uint32_t crossover = configs[0].unaligned->wg_denoms[1]; + return (n > crossover) ? 1 : 0; + } + uint32_t crossover_large = configs[1].unaligned->wg_denoms[1]; + if (n > crossover_large) return last; + uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1]; + if (n > crossover_medium) return 1; + return 0; + }; + } else { + device->matmul_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, + const std::vector<vk_matmul_pipeline_pair>& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + if (m <= 32 || n <= 32) return 0; + if (configs.size() == 2) return 1; + if (m <= 64 || n <= 64) return 1; + return (uint32_t)configs.size() - 1; + }; + device->matmul_id_tile_selector = device->matmul_tile_selector; } - if (buf->device != nullptr) { - buf->device->memory_logger->log_deallocation(buf); + // mul mat vec + + // the number of rows computed per shader depends on GPU model and quant + uint32_t rm_stdq = 1; + uint32_t rm_kq = 2; + uint32_t rm_stdq_int = 1; + uint32_t rm_kq_int = 1; + // Rows per workgroup for the BF16 mat-vec shader. It takes the + // iter_aligned_nonquant path, where after the rows-outside restructure each + // A load is fanned across all NUM_COLS columns. More rows per workgroup then + // amortizes the per-column B loads and the final reduction over more useful + // work, which is what the multi-column BF16 GEMV needs on GCN. Left at the + // stock 2 elsewhere so RDNA/NVIDIA/Intel keep byte-identical pipelines. + uint32_t rm_bf16 = 2; + auto const &rm_iq_int = [](uint32_t i) { return i == 0 ? 8u : 4u; }; + if (device->vendor_id == VK_VENDOR_ID_AMD) { + if (device->architecture == AMD_GCN) { + rm_stdq = 2; + rm_kq = 4; + rm_stdq_int = 4; + rm_bf16 = 4; + } + } else if (device->vendor_id == VK_VENDOR_ID_INTEL) { + rm_stdq = 2; + rm_stdq_int = 2; } + // RDNA3: above four columns, static 4 rows for all types bench faster than the default + const bool is_rdna3 = device->vendor_id == VK_VENDOR_ID_AMD && device->architecture == AMD_RDNA3; + auto const &rm_int_n = [&](uint32_t rows, uint32_t i) { return (is_rdna3 && i >= 4) ? 4u : rows; }; + // RDNA3: Static 4 rows for all types bench faster than the default + auto const &rm_id = [&](uint32_t rows) { return is_rdna3 ? 4u : rows; }; + uint32_t rm_iq = 2 * rm_kq; - buf.reset(); -} + // How wide a batch still belongs on the mat-vec path rather than mul_mm. + // GCN goes to 16: with the A load hoisted out of the column loop the GEMV + // cost is close to flat in NUM_COLS, so it stays well under the mul_mm + // crossover that otherwise bites at n=9. Every other architecture keeps the + // stock 8 -- both because the crossover has not been measured there and + // because each extra column instantiates another pipeline per type. + device->mul_mat_vec_max_cols_eff = mul_mat_vec_max_cols_default; + if (device->vendor_id == VK_VENDOR_ID_AMD && device->architecture == AMD_GCN) { + device->mul_mat_vec_max_cols_eff = 16; + } + // Escape hatch for A/B measurement without a rebuild. + if (const char * e = getenv("GGML_VK_MMV_MAX_COLS")) { + const int v = atoi(e); + if (v >= 1 && v <= (int)mul_mat_vec_max_cols) { + device->mul_mat_vec_max_cols_eff = (uint32_t)v; + } + } + // Escape hatch: rows per workgroup for K-quants / standard quants. + // rm changes workgroup ownership, not a row's k-reduction order. + if (const char * e = getenv("GGML_VK_MMV_ROWS_KQ")) { + const int v = atoi(e); + if (v >= 1 && v <= 64) { + rm_kq = (uint32_t)v; + rm_iq = 2 * rm_kq; + } + } + if (const char * e = getenv("GGML_VK_MMV_ROWS_STDQ")) { + const int v = atoi(e); + if (v >= 1 && v <= 64) { + rm_stdq = (uint32_t)v; + } + } + if (requested == nullptr) { + GGML_LOG_INFO("ggml_vulkan: %s mmv rows rm_kq=%u rm_stdq=%u\n", + device->name.c_str(), rm_kq, rm_stdq); + } -static vk_subbuffer ggml_vk_subbuffer(const ggml_backend_vk_context* ctx, const vk_buffer& buf, size_t offset = 0) { - return { buf, offset, ggml_vk_get_max_buffer_range(ctx, buf, offset) }; -} + const bool use_subgroups = device->subgroup_arithmetic; + // The Imagination proprietary compiler rejects the subgroup-only dequant mul_mat_vec + // shaders that require a subgroup size >= 16; fall back to shared-memory reduction. + const bool is_imagination_proprietary = + device->driver_id == vk::DriverId::eImaginationProprietary; + // Ensure a subgroup size >= 16 is available + const bool use_subgroups16 = use_subgroups && subgroup_min_size_16 && !is_imagination_proprietary; -static void ggml_vk_sync_buffers(ggml_backend_vk_context* ctx, vk_context& subctx) { - VK_LOG_DEBUG("ggml_vk_sync_buffers()"); + const uint32_t subgroup_size = (device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control && device->subgroup_min_size <= 16 && device->subgroup_max_size >= 16) ? 16 : device->subgroup_size; + const uint32_t subgroup_size16 = std::max(subgroup_size, 16u); - ggml_vk_flush_cpy_regions(subctx); + const uint32_t force_subgroup_size = use_subgroups ? subgroup_size : 0; + const uint32_t force_subgroup_size16 = use_subgroups16 ? subgroup_size16 : 0; + static constexpr uint32_t mul_mat_vec_num_bindings = 5; + // 6 stock + 1 for the weight-paging per-expert offset array (binding 6). + static constexpr uint32_t mul_mat_vec_id_num_bindings = 7; - const bool transfer_queue = subctx->p->q->transfer_only; +#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) +#define OCP_DMMV_LEN(NAME, REDUC) (device->ocp_fp4 ? NAME ## _ocp_len[REDUC] : NAME ## _len[REDUC]) +#define OCP_DMMV_DATA(NAME, REDUC) (device->ocp_fp4 ? NAME ## _ocp_data[REDUC] : NAME ## _data[REDUC]) +#else +#define OCP_DMMV_LEN(NAME, REDUC) NAME ## _len[REDUC] +#define OCP_DMMV_DATA(NAME, REDUC) NAME ## _data[REDUC] +#endif - if (ctx) { - ctx->prealloc_x_need_sync = ctx->prealloc_y_need_sync = ctx->prealloc_split_k_need_sync = false; - } + for (uint32_t w = 0; w < DMMV_WG_SIZE_COUNT; ++w) { + const uint32_t wg_size_subgroup = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size : (subgroup_size * 4); + const uint32_t wg_size_subgroup16 = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size16 : (subgroup_size16 * 4); - subctx->s->buffer->buf.pipelineBarrier( - subctx->p->q->stage_flags, - subctx->p->q->stage_flags, - {}, - { { - { !transfer_queue ? (vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite) : (vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite) }, - { !transfer_queue ? (vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite) : (vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eTransferWrite) } - } }, - {}, - {} - ); -} + const shader_reduction_mode reduc = (use_subgroups && w == DMMV_WG_SIZE_SUBGROUP) ? SHADER_REDUCTION_MODE_SUBGROUP : + (use_subgroups && w == DMMV_WG_SIZE_LARGE) ? SHADER_REDUCTION_MODE_HYBRID : + SHADER_REDUCTION_MODE_SHMEM; -static void ggml_vk_reset_event(vk_context& ctx, vk::Event& event) { - VK_LOG_DEBUG("ggml_vk_set_event()"); - - ctx->s->buffer->buf.resetEvent( - event, - ctx->p->q->stage_flags - ); -} - -static void ggml_vk_set_event(vk_context& ctx, vk::Event& event) { - VK_LOG_DEBUG("ggml_vk_set_event()"); - - ctx->s->buffer->buf.setEvent( - event, - ctx->p->q->stage_flags - ); -} - -static void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events) { - VK_LOG_DEBUG("ggml_vk_wait_events()"); - if (events.empty()) { - return; - } - - ctx->s->buffer->buf.waitEvents( - events, - ctx->p->q->stage_flags, - ctx->p->q->stage_flags, - {}, - {}, - {} - ); -} - -struct vk_fa_tuning_params { - FaCodePath path; - uint32_t workgroup_size; - uint32_t subgroup_size; - uint32_t block_rows; - uint32_t block_cols; - uint32_t d_split; - uint32_t row_split; - bool shmem_staging; - bool disable_subgroups; - uint32_t limit_occupancy_shmem; - - void print() const { - std::cerr << "path=" << path << " workgroup_size=" << workgroup_size << " subgroup_size=" << subgroup_size << - " block_rows=" << block_rows << " block_cols=" << block_cols << " d_split=" << d_split << - " row_split=" << row_split << " shmem_staging=" << shmem_staging << " disable_subgroups=" << disable_subgroups << - " limit_occupancy_shmem=" << limit_occupancy_shmem << std::endl; - } -}; - -static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type); -static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type = GGML_TYPE_F16, ggml_type v_type = GGML_TYPE_F16); + const shader_reduction_mode reduc16 = (use_subgroups16 && w == DMMV_WG_SIZE_SUBGROUP) ? SHADER_REDUCTION_MODE_SUBGROUP : + (use_subgroups16 && w == DMMV_WG_SIZE_LARGE) ? SHADER_REDUCTION_MODE_HYBRID : + SHADER_REDUCTION_MODE_SHMEM; -static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) { + for (uint32_t i = 0; i < device->mul_mat_vec_max_cols_eff; ++i) { + // Rows per workgroup for BF16, as a function of the batch width. + // After the rows-outside restructure the shader holds NUM_COLS B + // vec4s plus NUM_COLS*NUM_ROWS accumulators live at once, so the + // register footprint grows with the product of the two. Cap that + // product so wide batches do not spill and crater occupancy: on + // GCN this yields 4 rows up to 8 columns, tapering to 2 at 16. + // Where rm_bf16 is the stock 2 (everything except GCN) this clamp + // never binds, so those pipelines are unchanged. + const uint32_t rm_bf16_i = std::max(1u, std::min(rm_bf16, 32u / (i + 1))); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f32_f32", arr_dmmv_f32_f32_f32_len[reduc], arr_dmmv_f32_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1, 1, 1}, {wg_size_subgroup, 1, i+1}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f32_f32", arr_dmmv_f16_f32_f32_len[reduc], arr_dmmv_f16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f32_f32", arr_dmmv_bf16_f32_f32_len[reduc], arr_dmmv_bf16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_bf16_i, 1, 1}, {wg_size_subgroup, rm_bf16_i, i+1}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f32_f32", arr_dmmv_q1_0_f32_f32_len[reduc], arr_dmmv_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f32_f32", arr_dmmv_q2_0_f32_f32_len[reduc], arr_dmmv_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f32_f32", arr_dmmv_q4_0_f32_f32_len[reduc], arr_dmmv_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f32_f32", arr_dmmv_q4_1_f32_f32_len[reduc], arr_dmmv_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f32_f32", arr_dmmv_q5_0_f32_f32_len[reduc], arr_dmmv_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f32_f32", arr_dmmv_q5_1_f32_f32_len[reduc], arr_dmmv_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f32_f32", arr_dmmv_q8_0_f32_f32_len[reduc], arr_dmmv_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f32_f32", arr_dmmv_q2_k_f32_f32_len[reduc16], arr_dmmv_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f32_f32", arr_dmmv_q3_k_f32_f32_len[reduc16], arr_dmmv_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f32_f32", arr_dmmv_q4_k_f32_f32_len[reduc16], arr_dmmv_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f32_f32", arr_dmmv_q5_k_f32_f32_len[reduc16], arr_dmmv_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f32_f32", arr_dmmv_q6_k_f32_f32_len[reduc16], arr_dmmv_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f32_f32", arr_dmmv_tq1_0_f32_f32_len[reduc16], arr_dmmv_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f32_f32", arr_dmmv_tq2_0_f32_f32_len[reduc16], arr_dmmv_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f32_f32", arr_dmmv_iq1_s_f32_f32_len[reduc16], arr_dmmv_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f32_f32", arr_dmmv_iq1_m_f32_f32_len[reduc16], arr_dmmv_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f32_f32", arr_dmmv_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f32_f32", arr_dmmv_iq2_xs_f32_f32_len[reduc16], arr_dmmv_iq2_xs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f32_f32", arr_dmmv_iq2_s_f32_f32_len[reduc16], arr_dmmv_iq2_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f32_f32", arr_dmmv_iq3_xxs_f32_f32_len[reduc16], arr_dmmv_iq3_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f32_f32", arr_dmmv_iq3_s_f32_f32_len[reduc16], arr_dmmv_iq3_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f32_f32", arr_dmmv_iq4_xs_f32_f32_len[reduc16], arr_dmmv_iq4_xs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f32_f32", arr_dmmv_iq4_nl_f32_f32_len[reduc16], arr_dmmv_iq4_nl_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f32_f32", OCP_DMMV_LEN(arr_dmmv_mxfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_mxfp4_f32_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f32_f32", OCP_DMMV_LEN(arr_dmmv_nvfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_nvfp4_f32_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - vk_fa_tuning_params result{}; - result.path = FA_SCALAR; + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f16_f32", arr_dmmv_f32_f16_f32_len[reduc], arr_dmmv_f32_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1, 1, 1}, {wg_size_subgroup, 1, i+1}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f16_f32", arr_dmmv_f16_f16_f32_len[reduc], arr_dmmv_f16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f16_f32", arr_dmmv_bf16_f16_f32_len[reduc], arr_dmmv_bf16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_bf16_i, 1, 1}, {wg_size_subgroup, rm_bf16_i, i+1}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f16_f32", arr_dmmv_q1_0_f16_f32_len[reduc], arr_dmmv_q1_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f16_f32", arr_dmmv_q2_0_f16_f32_len[reduc], arr_dmmv_q2_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f16_f32", arr_dmmv_q4_0_f16_f32_len[reduc], arr_dmmv_q4_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f16_f32", arr_dmmv_q4_1_f16_f32_len[reduc], arr_dmmv_q4_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f16_f32", arr_dmmv_q5_0_f16_f32_len[reduc], arr_dmmv_q5_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f16_f32", arr_dmmv_q5_1_f16_f32_len[reduc], arr_dmmv_q5_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f16_f32", arr_dmmv_q8_0_f16_f32_len[reduc], arr_dmmv_q8_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f16_f32", arr_dmmv_q2_k_f16_f32_len[reduc16], arr_dmmv_q2_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f16_f32", arr_dmmv_q3_k_f16_f32_len[reduc16], arr_dmmv_q3_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f16_f32", arr_dmmv_q4_k_f16_f32_len[reduc16], arr_dmmv_q4_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f16_f32", arr_dmmv_q5_k_f16_f32_len[reduc16], arr_dmmv_q5_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f16_f32", arr_dmmv_q6_k_f16_f32_len[reduc16], arr_dmmv_q6_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f16_f32", arr_dmmv_tq1_0_f16_f32_len[reduc16], arr_dmmv_tq1_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f16_f32", arr_dmmv_tq2_0_f16_f32_len[reduc16], arr_dmmv_tq2_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f16_f32", arr_dmmv_iq1_s_f16_f32_len[reduc16], arr_dmmv_iq1_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f16_f32", arr_dmmv_iq1_m_f16_f32_len[reduc16], arr_dmmv_iq1_m_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f16_f32", arr_dmmv_iq2_xxs_f16_f32_len[reduc16], arr_dmmv_iq2_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f16_f32", arr_dmmv_iq2_xs_f16_f32_len[reduc16], arr_dmmv_iq2_xs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f16_f32", arr_dmmv_iq2_s_f16_f32_len[reduc16], arr_dmmv_iq2_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f16_f32", arr_dmmv_iq3_xxs_f16_f32_len[reduc16], arr_dmmv_iq3_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f16_f32", arr_dmmv_iq3_s_f16_f32_len[reduc16], arr_dmmv_iq3_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f16_f32", arr_dmmv_iq4_xs_f16_f32_len[reduc16], arr_dmmv_iq4_xs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f16_f32", arr_dmmv_iq4_nl_f16_f32_len[reduc16], arr_dmmv_iq4_nl_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f16_f32", OCP_DMMV_LEN(arr_dmmv_mxfp4_f16_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_mxfp4_f16_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f16_f32", OCP_DMMV_LEN(arr_dmmv_nvfp4_f16_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_nvfp4_f16_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - if (device->vendor_id == VK_VENDOR_ID_INTEL) { - // Disable subgroup use due to performance issues when enforcing subgroup sizes - result.subgroup_size = 32; - result.disable_subgroups = true; - } else if (device->vendor_id == VK_VENDOR_ID_AMD && device->architecture != AMD_GCN) { - result.subgroup_size = n_rows < 4 ? 32 : device->subgroup_size; - } else { - result.subgroup_size = device->subgroup_size; - } +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (device->integer_dot_product) { + const uint32_t subgroup_size_int = (device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control) ? device->subgroup_min_size : device->subgroup_size; + const uint32_t wg_size_subgroup_int = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size_int : (subgroup_size_int * 4); - // Row split splits the workgroup so that synchronization only has to happen within subgroups, which avoids barriers - uint32_t row_split_max_hsk = 64; - if (device->vendor_id == VK_VENDOR_ID_AMD && device->architecture != AMD_GCN && !device->uma) { - row_split_max_hsk = n_rows <= 8 ? 64 : 128; - } - result.row_split = (n_rows < 4 || hsk <= row_split_max_hsk) ? 1 : 4; + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_q8_1_f32", arr_dmmv_q2_0_q8_1_f32_len[reduc], arr_dmmv_q2_0_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(2*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(2*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_q8_1_f32", arr_dmmv_q4_0_q8_1_f32_len[reduc], arr_dmmv_q4_0_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_q8_1_f32", arr_dmmv_q4_1_q8_1_f32_len[reduc], arr_dmmv_q4_1_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_q8_1_f32", arr_dmmv_q5_0_q8_1_f32_len[reduc], arr_dmmv_q5_0_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_q8_1_f32", arr_dmmv_q5_1_q8_1_f32_len[reduc], arr_dmmv_q5_1_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_q8_1_f32", arr_dmmv_q8_0_q8_1_f32_len[reduc], arr_dmmv_q8_0_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - if (result.subgroup_size > 32 && (n_rows < 4 || hsk < (result.row_split == 1 ? 128 : 64))) { - result.workgroup_size = result.subgroup_size * 2; - } else { - result.workgroup_size = result.subgroup_size * 4; - } + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_q8_1_f32", arr_dmmv_mxfp4_q8_1_f32_len[reduc], arr_dmmv_mxfp4_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(2*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(2*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - const uint32_t D = hsk | hsv; + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_q8_1_f32", arr_dmmv_q2_k_q8_1_f32_len[reduc], arr_dmmv_q2_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(2*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(2*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_q8_1_f32", arr_dmmv_q3_k_q8_1_f32_len[reduc], arr_dmmv_q3_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_q8_1_f32", arr_dmmv_q4_k_q8_1_f32_len[reduc], arr_dmmv_q4_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_q8_1_f32", arr_dmmv_q5_k_q8_1_f32_len[reduc], arr_dmmv_q5_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_q8_1_f32", arr_dmmv_q6_k_q8_1_f32_len[reduc], arr_dmmv_q6_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - const bool reduce_block_rows = D & 8 || n_kv < 1024 || device->vendor_id == VK_VENDOR_ID_INTEL; + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_q8_1_f32", arr_dmmv_iq1_s_q8_1_f32_len[reduc], arr_dmmv_iq1_s_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_iq_int(i), 1, 1}, {wg_size_subgroup_int, 1*rm_iq_int(i), i+1}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_q8_1_f32", arr_dmmv_iq1_m_q8_1_f32_len[reduc], arr_dmmv_iq1_m_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_iq_int(i), 1, 1}, {wg_size_subgroup_int, 1*rm_iq_int(i), i+1}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_q8_1_f32", arr_dmmv_iq4_xs_q8_1_f32_len[reduc], arr_dmmv_iq4_xs_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - if (n_rows == 1) { - result.block_rows = 1; - result.block_cols = 64; - } else { - // row_split 1 means higher register use per row, so block size has to be adjusted - if (result.row_split == 1) { - result.block_rows = n_rows == 2 ? 2 : ((n_rows <= 4 || reduce_block_rows) ? 4 : 8); - } else { - result.block_rows = n_rows <= 4 ? 4 : ((n_rows <= 8 || reduce_block_rows) ? 8 : 16); + } +#endif // GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT } - result.block_cols = (D & 8) ? 64 : 32; - } + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F32 ], "mul_mat_vec_id_f32_f32", arr_dmmv_id_f32_f32_f32_len[reduc], arr_dmmv_id_f32_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1, 1, 1}, {wg_size_subgroup, 1}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F16 ], "mul_mat_vec_id_f16_f32", arr_dmmv_id_f16_f32_f32_len[reduc], arr_dmmv_id_f16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_BF16], "mul_mat_vec_id_bf16_f32", arr_dmmv_id_bf16_f32_f32_len[reduc], arr_dmmv_id_bf16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q1_0], "mul_mat_vec_id_q1_0_f32", arr_dmmv_id_q1_0_f32_f32_len[reduc], arr_dmmv_id_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_0], "mul_mat_vec_id_q2_0_f32", arr_dmmv_id_q2_0_f32_f32_len[reduc], arr_dmmv_id_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_0], "mul_mat_vec_id_q4_0_f32", arr_dmmv_id_q4_0_f32_f32_len[reduc], arr_dmmv_id_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_1], "mul_mat_vec_id_q4_1_f32", arr_dmmv_id_q4_1_f32_f32_len[reduc], arr_dmmv_id_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_0], "mul_mat_vec_id_q5_0_f32", arr_dmmv_id_q5_0_f32_f32_len[reduc], arr_dmmv_id_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_1], "mul_mat_vec_id_q5_1_f32", arr_dmmv_id_q5_1_f32_f32_len[reduc], arr_dmmv_id_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q8_0], "mul_mat_vec_id_q8_0_f32", arr_dmmv_id_q8_0_f32_f32_len[reduc], arr_dmmv_id_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_K], "mul_mat_vec_id_q2_k_f32", arr_dmmv_id_q2_k_f32_f32_len[reduc16], arr_dmmv_id_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q3_K], "mul_mat_vec_id_q3_k_f32", arr_dmmv_id_q3_k_f32_f32_len[reduc16], arr_dmmv_id_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_K], "mul_mat_vec_id_q4_k_f32", arr_dmmv_id_q4_k_f32_f32_len[reduc16], arr_dmmv_id_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_K], "mul_mat_vec_id_q5_k_f32", arr_dmmv_id_q5_k_f32_f32_len[reduc16], arr_dmmv_id_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q6_K], "mul_mat_vec_id_q6_k_f32", arr_dmmv_id_q6_k_f32_f32_len[reduc16], arr_dmmv_id_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ1_0], "mul_mat_vec_id_tq1_0_f32", arr_dmmv_id_tq1_0_f32_f32_len[reduc16], arr_dmmv_id_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ2_0], "mul_mat_vec_id_tq2_0_f32", arr_dmmv_id_tq2_0_f32_f32_len[reduc16], arr_dmmv_id_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_f32", arr_dmmv_id_iq1_s_f32_f32_len[reduc16], arr_dmmv_id_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_f32", arr_dmmv_id_iq1_m_f32_f32_len[reduc16], arr_dmmv_id_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XXS], "mul_mat_vec_id_iq2_xxs_f32", arr_dmmv_id_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XS], "mul_mat_vec_id_iq2_xs_f32", arr_dmmv_id_iq2_xs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_S], "mul_mat_vec_id_iq2_s_f32", arr_dmmv_id_iq2_s_f32_f32_len[reduc16], arr_dmmv_id_iq2_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_XXS], "mul_mat_vec_id_iq3_xxs_f32", arr_dmmv_id_iq3_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq3_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_S], "mul_mat_vec_id_iq3_s_f32", arr_dmmv_id_iq3_s_f32_f32_len[reduc16], arr_dmmv_id_iq3_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_XS], "mul_mat_vec_id_iq4_xs_f32", arr_dmmv_id_iq4_xs_f32_f32_len[reduc16], arr_dmmv_id_iq4_xs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_NL], "mul_mat_vec_id_iq4_nl_f32", arr_dmmv_id_iq4_nl_f32_f32_len[reduc16], arr_dmmv_id_iq4_nl_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_MXFP4], "mul_mat_vec_id_mxfp4_f32", OCP_DMMV_LEN(arr_dmmv_id_mxfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_id_mxfp4_f32_f32, reduc16), "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_NVFP4], "mul_mat_vec_id_nvfp4_f32", OCP_DMMV_LEN(arr_dmmv_id_nvfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_id_nvfp4_f32_f32, reduc16), "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - const uint32_t D_lsb = D ^ (D & (D-1)); // extract lowest set bit +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (device->integer_dot_product) { + const uint32_t subgroup_size_int = (device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control) ? device->subgroup_min_size : device->subgroup_size; + const uint32_t wg_size_subgroup_int = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size_int : (subgroup_size_int * 4); - result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q2_0], "mul_mat_vec_id_q2_0_q8_1_f32", arr_dmmv_id_q2_0_q8_1_f32_len[reduc], arr_dmmv_id_q2_0_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(2*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(2*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q4_0], "mul_mat_vec_id_q4_0_q8_1_f32", arr_dmmv_id_q4_0_q8_1_f32_len[reduc], arr_dmmv_id_q4_0_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q4_1], "mul_mat_vec_id_q4_1_q8_1_f32", arr_dmmv_id_q4_1_q8_1_f32_len[reduc], arr_dmmv_id_q4_1_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q5_0], "mul_mat_vec_id_q5_0_q8_1_f32", arr_dmmv_id_q5_0_q8_1_f32_len[reduc], arr_dmmv_id_q5_0_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q5_1], "mul_mat_vec_id_q5_1_q8_1_f32", arr_dmmv_id_q5_1_q8_1_f32_len[reduc], arr_dmmv_id_q5_1_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q8_0], "mul_mat_vec_id_q8_0_q8_1_f32", arr_dmmv_id_q8_0_q8_1_f32_len[reduc], arr_dmmv_id_q8_0_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); - result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0; + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_MXFP4], "mul_mat_vec_id_mxfp4_q8_1_f32", arr_dmmv_id_mxfp4_q8_1_f32_len[reduc], arr_dmmv_id_mxfp4_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(2*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(2*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); - if (!reduce_block_rows && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) { - result.block_rows /= 2; - } + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q2_K], "mul_mat_vec_id_q2_k_q8_1_f32", arr_dmmv_id_q2_k_q8_1_f32_len[reduc], arr_dmmv_id_q2_k_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(2*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(2*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q3_K], "mul_mat_vec_id_q3_k_q8_1_f32", arr_dmmv_id_q3_k_q8_1_f32_len[reduc], arr_dmmv_id_q3_k_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q4_K], "mul_mat_vec_id_q4_k_q8_1_f32", arr_dmmv_id_q4_k_q8_1_f32_len[reduc], arr_dmmv_id_q4_k_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q5_K], "mul_mat_vec_id_q5_k_q8_1_f32", arr_dmmv_id_q5_k_q8_1_f32_len[reduc], arr_dmmv_id_q5_k_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q6_K], "mul_mat_vec_id_q6_k_q8_1_f32", arr_dmmv_id_q6_k_q8_1_f32_len[reduc], arr_dmmv_id_q6_k_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); - // On AMD RDNA, for small head sizes and big batch size the shader uses few registers, so too many subgroups get scheduled - // at once and end up thrashing the cache. Fix this by setting a large (unused) shmem buffer that reduces occupancy. - // This targets an occupancy of 4 subgroups per SIMD. - if (device->vendor_id == VK_VENDOR_ID_AMD && device->properties.limits.maxComputeSharedMemorySize == 65536) { - if (device->architecture != AMD_GCN && n_rows >= 64 && hsk <= 128) { - // 30kb target for hsk > 64, 26kb for <= 64 due to smaller workgroup size - // Values are guessed, tested on RDNA2 - result.limit_occupancy_shmem = (hsk <= 64 ? 26 : 30) * 1024 / 4 / 4; - } else if (device->architecture == AMD_GCN && n_rows <= 8 && hsk >= 256) { - // Same thing for GCN, with an occupancy target of 2 subgroups per SIMD. - // Here low-batch FA with large head size is affected. - // n_rows < 4 switch because workgroup size switches from 128 to 256 there. - result.limit_occupancy_shmem = (n_rows < 4 ? 14 : 26) * 1024 / 4 / 4; + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_q8_1_f32", arr_dmmv_id_iq1_s_q8_1_f32_len[reduc], arr_dmmv_id_iq1_s_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*rm_iq_int(0), 1, 1}, {wg_size_subgroup_int, 1*rm_iq_int(0)}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_q8_1_f32", arr_dmmv_id_iq1_m_q8_1_f32_len[reduc], arr_dmmv_id_iq1_m_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*rm_iq_int(0), 1, 1}, {wg_size_subgroup_int, 1*rm_iq_int(0)}, 1, true, use_subgroups, subgroup_size_int); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_IQ4_XS], "mul_mat_vec_id_iq4_xs_q8_1_f32", arr_dmmv_id_iq4_xs_q8_1_f32_len[reduc], arr_dmmv_id_iq4_xs_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); } +#endif // GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT } - return result; -} - -static vk_fa_tuning_params get_fa_tuning_params_coopmat1(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) { - GGML_UNUSED(n_rows); - GGML_UNUSED(n_kv); - GGML_UNUSED(k_type); - GGML_UNUSED(v_type); - GGML_UNUSED(f32acc); - - vk_fa_tuning_params result{}; - result.path = FA_COOPMAT1; - - const uint32_t D = hsk | hsv; - - const uint32_t coopmat_block_rows = 16; - const uint32_t coopmat_block_cols = 16; +#undef OCP_DMMV_DATA +#undef OCP_DMMV_LEN - const uint32_t num_subgroups = 4; +#if !defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + GGML_UNUSED(rm_stdq_int); + GGML_UNUSED(rm_kq_int); + GGML_UNUSED(is_rdna3); + GGML_UNUSED(rm_int_n); + GGML_UNUSED(rm_id); + GGML_UNUSED(rm_iq_int); +#endif - result.block_rows = coopmat_block_rows; - result.block_cols = coopmat_block_cols * num_subgroups; - result.row_split = num_subgroups; - result.subgroup_size = device->subgroup_size; - result.workgroup_size = num_subgroups * result.subgroup_size; + // dequant shaders + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_F32 ], "f32_to_f16", dequant_f32_len, dequant_f32_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q1_0], "dequant_q1_0", dequant_q1_0_len, dequant_q1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 8, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q2_0], "dequant_q2_0", dequant_q2_0_len, dequant_q2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_0], "dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_1], "dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_0], "dequant_q5_0", dequant_q5_0_len, dequant_q5_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_1], "dequant_q5_1", dequant_q5_1_len, dequant_q5_1_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q8_0], "dequant_q8_0", dequant_q8_0_len, dequant_q8_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant_transpose[GGML_TYPE_Q8_0], "dequant_q8_0_transpose", dequant_q8_0_transpose_len, dequant_q8_0_transpose_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q2_K], "dequant_q2_k", dequant_q2_k_len, dequant_q2_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q3_K], "dequant_q3_k", dequant_q3_k_len, dequant_q3_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_K], "dequant_q4_k", dequant_q4_k_len, dequant_q4_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_K], "dequant_q5_k", dequant_q5_k_len, dequant_q5_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q6_K], "dequant_q6_k", dequant_q6_k_len, dequant_q6_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ1_0], "dequant_tq1_0", dequant_tq1_0_len, dequant_tq1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ2_0], "dequant_tq2_0", dequant_tq2_0_len, dequant_tq2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ1_S], "dequant_iq1_s", dequant_iq1_s_len, dequant_iq1_s_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ1_M], "dequant_iq1_m", dequant_iq1_m_len, dequant_iq1_m_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ2_XXS], "dequant_iq2_xxs", dequant_iq2_xxs_len, dequant_iq2_xxs_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ2_XS], "dequant_iq2_xs", dequant_iq2_xs_len, dequant_iq2_xs_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ2_S], "dequant_iq2_s", dequant_iq2_s_len, dequant_iq2_s_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ3_XXS], "dequant_iq3_xxs", dequant_iq3_xxs_len, dequant_iq3_xxs_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ3_S], "dequant_iq3_s", dequant_iq3_s_len, dequant_iq3_s_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ4_XS], "dequant_iq4_xs", dequant_iq4_xs_len, dequant_iq4_xs_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ4_NL], "dequant_iq4_nl", dequant_iq4_nl_len, dequant_iq4_nl_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_MXFP4], "dequant_mxfp4", dequant_mxfp4_len, dequant_mxfp4_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_NVFP4], "dequant_nvfp4", dequant_nvfp4_len, dequant_nvfp4_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); - const uint32_t D_lsb = D ^ (D & (D-1)); // extract lowest set bit - result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4); + // get_rows + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_F32 ], "get_rows_f32", get_rows_f32_len, get_rows_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_F16 ], "get_rows_f16", get_rows_f16_len, get_rows_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_BF16], "get_rows_bf16", get_rows_bf16_len, get_rows_bf16_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q1_0], "get_rows_q1_0", get_rows_q1_0_len, get_rows_q1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q2_0], "get_rows_q2_0", get_rows_q2_0_len, get_rows_q2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_0], "get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_1], "get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_0], "get_rows_q5_0", get_rows_q5_0_len, get_rows_q5_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_1], "get_rows_q5_1", get_rows_q5_1_len, get_rows_q5_1_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q8_0], "get_rows_q8_0", get_rows_q8_0_len, get_rows_q8_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q2_K], "get_rows_q2_k", get_rows_q2_k_len, get_rows_q2_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q3_K], "get_rows_q3_k", get_rows_q3_k_len, get_rows_q3_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_K], "get_rows_q4_k", get_rows_q4_k_len, get_rows_q4_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_K], "get_rows_q5_k", get_rows_q5_k_len, get_rows_q5_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q6_K], "get_rows_q6_k", get_rows_q6_k_len, get_rows_q6_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TQ1_0], "get_rows_tq1_0", get_rows_tq1_0_len, get_rows_tq1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TQ2_0], "get_rows_tq2_0", get_rows_tq2_0_len, get_rows_tq2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ1_S], "get_rows_iq1_s", get_rows_iq1_s_len, get_rows_iq1_s_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ1_M], "get_rows_iq1_m", get_rows_iq1_m_len, get_rows_iq1_m_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ2_XXS], "get_rows_iq2_xxs", get_rows_iq2_xxs_len, get_rows_iq2_xxs_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ2_XS], "get_rows_iq2_xs", get_rows_iq2_xs_len, get_rows_iq2_xs_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ2_S], "get_rows_iq2_s", get_rows_iq2_s_len, get_rows_iq2_s_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ3_XXS], "get_rows_iq3_xxs", get_rows_iq3_xxs_len, get_rows_iq3_xxs_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ3_S], "get_rows_iq3_s", get_rows_iq3_s_len, get_rows_iq3_s_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ4_XS], "get_rows_iq4_xs", get_rows_iq4_xs_len, get_rows_iq4_xs_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ4_NL], "get_rows_iq4_nl", get_rows_iq4_nl_len, get_rows_iq4_nl_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_MXFP4], "get_rows_mxfp4", get_rows_mxfp4_len, get_rows_mxfp4_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_NVFP4], "get_rows_nvfp4", get_rows_nvfp4_len, get_rows_nvfp4_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_I32], "get_rows_i32", get_rows_i32_len, get_rows_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0; + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_F32 ], "get_rows_f32_f32", get_rows_f32_f32_len, get_rows_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_F16 ], "get_rows_f16_f32", get_rows_f16_f32_len, get_rows_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_BF16], "get_rows_bf16_f32", get_rows_bf16_f32_len, get_rows_bf16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q1_0], "get_rows_q1_0_f32", get_rows_q1_0_f32_len, get_rows_q1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q2_0], "get_rows_q2_0_f32", get_rows_q2_0_f32_len, get_rows_q2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_0], "get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_1], "get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_0], "get_rows_q5_0_f32", get_rows_q5_0_f32_len, get_rows_q5_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_1], "get_rows_q5_1_f32", get_rows_q5_1_f32_len, get_rows_q5_1_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q8_0], "get_rows_q8_0_f32", get_rows_q8_0_f32_len, get_rows_q8_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q2_K], "get_rows_q2_k_f32", get_rows_q2_k_f32_len, get_rows_q2_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q3_K], "get_rows_q3_k_f32", get_rows_q3_k_f32_len, get_rows_q3_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_K], "get_rows_q4_k_f32", get_rows_q4_k_f32_len, get_rows_q4_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_K], "get_rows_q5_k_f32", get_rows_q5_k_f32_len, get_rows_q5_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q6_K], "get_rows_q6_k_f32", get_rows_q6_k_f32_len, get_rows_q6_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TQ1_0], "get_rows_tq1_0_f32", get_rows_tq1_0_f32_len, get_rows_tq1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TQ2_0], "get_rows_tq2_0_f32", get_rows_tq2_0_f32_len, get_rows_tq2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ1_S], "get_rows_iq1_s_f32", get_rows_iq1_s_f32_len, get_rows_iq1_s_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ1_M], "get_rows_iq1_m_f32", get_rows_iq1_m_f32_len, get_rows_iq1_m_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ2_XXS], "get_rows_iq2_xxs_f32", get_rows_iq2_xxs_f32_len, get_rows_iq2_xxs_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ2_XS], "get_rows_iq2_xs_f32", get_rows_iq2_xs_f32_len, get_rows_iq2_xs_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ2_S], "get_rows_iq2_s_f32", get_rows_iq2_s_f32_len, get_rows_iq2_s_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ3_XXS], "get_rows_iq3_xxs_f32", get_rows_iq3_xxs_f32_len, get_rows_iq3_xxs_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ3_S], "get_rows_iq3_s_f32", get_rows_iq3_s_f32_len, get_rows_iq3_s_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ4_XS], "get_rows_iq4_xs_f32", get_rows_iq4_xs_f32_len, get_rows_iq4_xs_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ4_NL], "get_rows_iq4_nl_f32", get_rows_iq4_nl_f32_len, get_rows_iq4_nl_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_MXFP4], "get_rows_mxfp4_f32", get_rows_mxfp4_f32_len, get_rows_mxfp4_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_NVFP4], "get_rows_nvfp4_f32", get_rows_nvfp4_f32_len, get_rows_nvfp4_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_back_f32, "get_rows_back_f32", get_rows_back_f32_len, get_rows_back_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {256, 1, 1}, {}, 1, true); - return result; -} + ggml_vk_create_pipeline(device, device->pipeline_matmul_split_k_reduce, "split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_flash_attn_split_k_reduce, "fa_split_k_reduce", fa_split_k_reduce_len, fa_split_k_reduce_data, "main", 3, sizeof(vk_op_flash_attn_split_k_reduce_push_constants), {1, device->subgroup_size, 1}, {device->subgroup_size}, 1, true); -static vk_fa_tuning_params get_fa_tuning_params_coopmat2(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) { - GGML_UNUSED(n_kv); - GGML_UNUSED(f32acc); + if (device->vendor_id == VK_VENDOR_ID_INTEL && (device->architecture == INTEL_XE2 || (device->architecture == INTEL_XE1 && device->coopmat_support && device->uma))) { + auto upper_power_of_2 = [&](uint32_t in) { + GGML_ASSERT(in != 0); + if (in <= 1) return 1u; + uint32_t ret = in - 1; + ret |= ret >> 1; + ret |= ret >> 2; + ret |= ret >> 4; + ret |= ret >> 8; + ret |= ret >> 16; + return ret + 1; + }; - vk_fa_tuning_params result{}; - result.path = FA_COOPMAT2; + uint32_t xe_native_sub_group_size = 16; + if (device->architecture == INTEL_XE1) { + xe_native_sub_group_size = 8; + } + + for (auto& it : device->pipeline_xe_fa_decode_dual_phases) { + const uint32_t split_p_chunk = 32; + auto HdQk = it.first; + auto& pipelines = it.second; + uint32_t head_dim_qk = std::get<0>(HdQk); + uint32_t head_dim_pv = std::get<1>(HdQk); + uint32_t gqa_ratio = std::get<2>(HdQk); + uint32_t q_len = std::get<3>(HdQk); + const uint32_t out_dim_per_wg = gqa_ratio > 16 ? 8 : 16; + uint32_t aligned_q_len = upper_power_of_2(q_len); + uint32_t group_sz_ph1 = std::min(std::max(aligned_q_len * xe_native_sub_group_size, 64u), 256u); + uint32_t out_per_wg_ph1 = std::min(q_len, 256u / xe_native_sub_group_size); + uint32_t aligned_gqa_ratio = upper_power_of_2(gqa_ratio); + uint32_t split_p_per_iter_ph2 = 256; + uint32_t split_p_per_warp = 16; + uint32_t group_sz_ph2 = (split_p_per_iter_ph2 / split_p_per_warp) * xe_native_sub_group_size; + uint32_t out_per_wg_ph2 = std::min(std::max(16u / aligned_gqa_ratio, 1u), q_len); + ggml_vk_create_pipeline(device, pipelines.first, "xe_fa_decode_ph1", fa_decode_ph1_cm1_len, fa_decode_ph1_cm1_data, "main", 5, sizeof(vk_fa_xe_opt_push_constants), { 1, 32, 1 }, { group_sz_ph1, gqa_ratio, head_dim_qk, xe_native_sub_group_size, split_p_chunk, out_per_wg_ph1 }, 1, false, true, xe_native_sub_group_size); + ggml_vk_create_pipeline(device, pipelines.second, "xe_fa_decode_ph2", fa_decode_ph2_cm1_len, fa_decode_ph2_cm1_data, "main", 5, sizeof(vk_fa_xe_opt_push_constants), { 1, 1, 1 }, { group_sz_ph2, gqa_ratio, head_dim_pv, out_per_wg_ph2, xe_native_sub_group_size, split_p_per_iter_ph2, split_p_chunk, out_dim_per_wg }, 1, false, true, xe_native_sub_group_size); + } + } - const uint32_t D = hsk | hsv; + for (auto &it : device->pipeline_fa_mask_opt) { + auto BrBc = it.first; + ggml_vk_create_pipeline(device, it.second, "fa_mask_opt", fa_mask_opt_len, fa_mask_opt_data, "main", 2, sizeof(vk_op_flash_attn_mask_opt_push_constants), {1, 1, 1}, {128, 128 / device->subgroup_size, BrBc.first, BrBc.second}, 1, true, true, device->subgroup_size); + } - const bool small_rows = n_rows < 32; + { + // Large workgroup so the per-row KV scan parallelizes; capped to device limits. + const uint32_t compact_max = std::min({1024u, device->properties.limits.maxComputeWorkGroupInvocations, device->properties.limits.maxComputeWorkGroupSize[0]}); + + // Fast ballot prefix-sum path when the device supports full subgroups; otherwise + // a shared-memory prefix-sum fallback. Both emit a deterministic ascending list. + device->fa_sparse_compact_use_subgroups = device->subgroup_ballot && device->subgroup_require_full_support; + if (device->fa_sparse_compact_use_subgroups) { + const uint32_t compact_wg = std::max(device->subgroup_size, (compact_max / device->subgroup_size) * device->subgroup_size); + const uint32_t compact_num_sg = compact_wg / device->subgroup_size; + ggml_vk_create_pipeline(device, device->pipeline_fa_sparse_compact_subgroup, "fa_sparse_compact_subgroup", fa_sparse_compact_subgroup_len, fa_sparse_compact_subgroup_data, "main", 2, sizeof(vk_op_flash_attn_sparse_compact_push_constants), {1, 1, 1}, {compact_wg, compact_num_sg}, 1, true, true, device->subgroup_size); + } else { + ggml_vk_create_pipeline(device, device->pipeline_fa_sparse_compact, "fa_sparse_compact", fa_sparse_compact_len, fa_sparse_compact_data, "main", 2, sizeof(vk_op_flash_attn_sparse_compact_push_constants), {1, 1, 1}, {compact_max}, 1, true); + } + } - if (small_rows) { - result.block_rows = 32; - result.block_cols = 32; - } else if (ggml_is_quantized(k_type) || ggml_is_quantized(v_type) || hsk >= 256 || hsv >= 256) { - result.block_rows = (hsk >= 512 || hsv >= 512) ? 32 : 64; - result.block_cols = 32; + if (device->subgroup_clustered && device->subgroup_require_full_support) { + ggml_vk_create_pipeline(device, device->pipeline_quantize_q8_1_x4, "quantize_q8_1_x4", quantize_q8_1_x4_subgroup_len, quantize_q8_1_x4_subgroup_data, "main", 2, sizeof(vk_quantize_q8_1_push_constants), {32 * device->subgroup_size / 8, 1, 1}, { device->subgroup_size }, 1, true, true); } else { - result.block_rows = 64; - result.block_cols = 64; + ggml_vk_create_pipeline(device, device->pipeline_quantize_q8_1_x4, "quantize_q8_1_x4", quantize_q8_1_x4_len, quantize_q8_1_x4_data, "main", 2, sizeof(vk_quantize_q8_1_push_constants), {32 * device->subgroup_size / 8, 1, 1}, { device->subgroup_size }, 1); } - result.subgroup_size = device->subgroup_size; - result.workgroup_size = (small_rows && (D % 32) == 0) ? 256 : 128; - - return result; -} + for (uint32_t i = 0; i < p021_max_gqa_ratio; ++i) { + if (device->subgroup_arithmetic && device->subgroup_require_full_support) { + ggml_vk_create_pipeline2(device, device->pipeline_mul_mat_vec_p021_f16_f32[i], "mul_mat_vec_p021_f16_f32"+std::to_string(i+1), mul_mat_vec_p021_f16_f32_subgroup_add_len, mul_mat_vec_p021_f16_f32_subgroup_add_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_p021_push_constants), {1, 1, 1}, {device->subgroup_size, i + 1}, 1, true, true); + } else { + ggml_vk_create_pipeline2(device, device->pipeline_mul_mat_vec_p021_f16_f32[i], "mul_mat_vec_p021_f16_f32"+std::to_string(i+1), mul_mat_vec_p021_f16_f32_len, mul_mat_vec_p021_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_p021_push_constants), {1, 1, 1}, {device->subgroup_size, i + 1}, 1, true); + } + } + ggml_vk_create_pipeline(device, device->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_nc_push_constants), {1, 1, 1}, {}, 1); -static vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) { - FaCodePath path = device->coopmat2 ? FA_COOPMAT2 : - device->coopmat1_fa_support ? FA_COOPMAT1 : FA_SCALAR; + ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_turbo_wht_f32, "turbo_wht_f32", turbo_wht_f32_len, turbo_wht_f32_data, "main", 3, sizeof(vk_op_turbo_wht_push_constants), {1, 1, 1}, {}, 1); - if (path == FA_COOPMAT2 && k_type == GGML_TYPE_BF16 && !device->coopmat2_bf16_support) { - path = FA_COOPMAT1; - } - if (path == FA_COOPMAT1 && k_type == GGML_TYPE_BF16 && !device->coopmat_bf16_support) { - path = FA_SCALAR; - } + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_f32, "rms_norm_mul_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_f32, "rms_norm_mul_add_f32", rms_norm_mul_add_f32_len, rms_norm_mul_add_f32_data, "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_f32, "rms_norm_mul_add_mul_f32", rms_norm_mul_add_f32_len, rms_norm_mul_add_f32_data, "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_partials_f32, "rms_norm_mul_add_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_partials_f32, "rms_norm_mul_add_mul_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f32, "rms_norm_set_rows_f32_f32", rms_norm_set_rows_f32_f32_len, rms_norm_set_rows_f32_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f16, "rms_norm_set_rows_f32_f16", rms_norm_set_rows_f32_f16_len, rms_norm_set_rows_f32_f16_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_partials_f32, "rms_norm_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_partials_f32, "rms_norm_mul_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true); - if (path == FA_COOPMAT1 && device->architecture == vk_device_architecture::NVIDIA_TURING) { - // Nvidia compiler bug, see https://github.com/ggml-org/llama.cpp/pull/19075#issuecomment-3820716090 - path = FA_SCALAR; + if (sizeof(vk_op_rms_norm_mul_rope_push_constants) <= device->properties.limits.maxPushConstantsSize) { + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f32, "rms_norm_mul_rope_f32_f32", rms_norm_mul_rope_f32_f32_len, rms_norm_mul_rope_f32_f32_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f16, "rms_norm_mul_rope_f32_f16", rms_norm_mul_rope_f32_f16_len, rms_norm_mul_rope_f32_f16_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true); } - if (path == FA_COOPMAT1) { - bool shape_ok = (f32acc && device->coopmat_support_16x16x16_f32acc) || - (!f32acc && device->coopmat_support_16x16x16_f16acc); - const vk_fa_tuning_params params = get_fa_tuning_params_coopmat1(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc); - bool shmem_ok = ggml_vk_flash_attn_coopmat_shmem_support(device, params, hsk, hsv, f32acc, k_type, v_type); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_sinkhorn_norm_f32[0], "sinkhorn_norm_f32_n2", sinkhorn_norm_f32_n2_len, sinkhorn_norm_f32_n2_data, "main", 2, sizeof(vk_op_sinkhorn_norm_push_constants), {64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_sinkhorn_norm_f32[1], "sinkhorn_norm_f32_n4", sinkhorn_norm_f32_n4_len, sinkhorn_norm_f32_n4_data, "main", 2, sizeof(vk_op_sinkhorn_norm_push_constants), {64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_sinkhorn_norm_f32[2], "sinkhorn_norm_f32_n8", sinkhorn_norm_f32_n8_len, sinkhorn_norm_f32_n8_data, "main", 2, sizeof(vk_op_sinkhorn_norm_push_constants), {64, 1, 1}, {}, 1); - if (!shape_ok || !shmem_ok) { - path = FA_SCALAR; - } - } + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f32, "cpy_f32_f32", cpy_f32_f32_len, cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f16_f16, "cpy_f16_f16", cpy_f16_f16_len, cpy_f16_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f16_f32, "cpy_f16_f32", cpy_f16_f32_len, cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_bf16,"cpy_f32_bf16",cpy_f32_bf16_len,cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_bf16_f32,"cpy_bf16_f32",cpy_bf16_f32_len,cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_i32_f32, "cpy_i32_f32", cpy_i32_f32_len, cpy_i32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_i32, "cpy_f32_i32", cpy_f32_i32_len, cpy_f32_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - // scalar is faster than coopmat when N==1 - if (n_rows == 1 && (path == FA_COOPMAT1 || path == FA_COOPMAT2)) { - path = FA_SCALAR; - } + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_f32, "contig_cpy_f32_f32", contig_cpy_f32_f32_len, contig_cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_f16, "contig_cpy_f32_f16", contig_cpy_f32_f16_len, contig_cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f16_f16, "contig_cpy_f16_f16", contig_cpy_f16_f16_len, contig_cpy_f16_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f16_f32, "contig_cpy_f16_f32", contig_cpy_f16_f32_len, contig_cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_bf16,"contig_cpy_f32_bf16",contig_cpy_f32_bf16_len,contig_cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_bf16_f32,"contig_cpy_bf16_f32",contig_cpy_bf16_f32_len,contig_cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_i32_f32, "contig_cpy_i32_f32", contig_cpy_i32_f32_len, contig_cpy_i32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_i32, "contig_cpy_f32_i32", contig_cpy_f32_i32_len, contig_cpy_f32_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - switch (path) { - case FA_SCALAR: - return get_fa_tuning_params_scalar(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc); - case FA_COOPMAT1: - return get_fa_tuning_params_coopmat1(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc); - case FA_COOPMAT2: - return get_fa_tuning_params_coopmat2(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc); - default: - throw std::runtime_error("unsupported FaCodePath"); - } -} + ggml_vk_create_pipeline(device, device->pipeline_cpy_transpose_32, "cpy_transpose_32", cpy_transpose_32_len, cpy_transpose_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_transpose_16, "cpy_transpose_16", cpy_transpose_16_len, cpy_transpose_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_transpose_02_32, "cpy_transpose_02_32", cpy_transpose_02_32_len, cpy_transpose_02_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_transpose_02_16, "cpy_transpose_02_16", cpy_transpose_02_16_len, cpy_transpose_02_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); -static vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc, - bool use_mask, bool use_mask_opt, bool use_logit_softcap, ggml_type k_type, ggml_type v_type) { - const bool old_amd_windows = device->vendor_id == VK_VENDOR_ID_AMD && device->driver_id == vk::DriverId::eAmdProprietary && - (device->architecture == AMD_GCN || device->architecture == AMD_RDNA1 || device->architecture == AMD_RDNA2); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q1_0], "cpy_f32_q1_0", cpy_f32_q1_0_len, cpy_f32_q1_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q2_0], "cpy_f32_q2_0", cpy_f32_q2_0_len, cpy_f32_q2_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q4_0], "cpy_f32_q4_0", cpy_f32_q4_0_len, cpy_f32_q4_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q4_1], "cpy_f32_q4_1", cpy_f32_q4_1_len, cpy_f32_q4_1_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q5_0], "cpy_f32_q5_0", cpy_f32_q5_0_len, cpy_f32_q5_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q5_1], "cpy_f32_q5_1", cpy_f32_q5_1_len, cpy_f32_q5_1_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q8_0], "cpy_f32_q8_0", cpy_f32_q8_0_len, cpy_f32_q8_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_IQ4_NL], "cpy_f32_iq4_nl", cpy_f32_iq4_nl_len, cpy_f32_iq4_nl_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); + // turbo4_0: 128 threads/workgroup, one workgroup per 128-element block; wg_denoms={1,1,1} + // so elements[0] (= n_blocks after CEIL_DIV by 128) maps 1-to-1 to workgroup count. + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_TURBO4_0], "cpy_f32_turbo4_0", cpy_f32_turbo4_0_len, cpy_f32_turbo4_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); - uint32_t flags = (use_mask_opt ? 1 : 0) | - (use_mask ? 2 : 0) | - (use_logit_softcap ? 4 : 0) | - (old_amd_windows ? 8 : 0); +#define SET_ROWS(src_idx, src, itype) \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_F32], "set_rows_" #src "_f32" #itype, set_rows_ ## src ## _f32 ## itype ## _len, set_rows_ ## src ## _f32 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_F16], "set_rows_" #src "_f16" #itype, set_rows_ ## src ## _f16 ## itype ## _len, set_rows_ ## src ## _f16 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_BF16], "set_rows_" #src "_bf16" #itype, set_rows_ ## src ## _bf16 ## itype ## _len, set_rows_ ## src ## _bf16 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q1_0], "set_rows_" #src "_q1_0" #itype, set_rows_ ## src ## _q1_0 ## itype ## _len, set_rows_ ## src ## _q1_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q2_0], "set_rows_" #src "_q2_0" #itype, set_rows_ ## src ## _q2_0 ## itype ## _len, set_rows_ ## src ## _q2_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q4_0], "set_rows_" #src "_q4_0" #itype, set_rows_ ## src ## _q4_0 ## itype ## _len, set_rows_ ## src ## _q4_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q4_1], "set_rows_" #src "_q4_1" #itype, set_rows_ ## src ## _q4_1 ## itype ## _len, set_rows_ ## src ## _q4_1 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q5_0], "set_rows_" #src "_q5_0" #itype, set_rows_ ## src ## _q5_0 ## itype ## _len, set_rows_ ## src ## _q5_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q5_1], "set_rows_" #src "_q5_1" #itype, set_rows_ ## src ## _q5_1 ## itype ## _len, set_rows_ ## src ## _q5_1 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q8_0], "set_rows_" #src "_q8_0" #itype, set_rows_ ## src ## _q8_0 ## itype ## _len, set_rows_ ## src ## _q8_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_IQ4_NL], "set_rows_" #src "_iq4_nl" #itype, set_rows_ ## src ## _iq4_nl ## itype ## _len, set_rows_ ## src ## _iq4_nl ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); - const uint32_t subgroup_size = params.disable_subgroups ? 0 : params.subgroup_size; + SET_ROWS(0, f32, _i32) + SET_ROWS(0, f32, _i64) + SET_ROWS(1, f16, _i32) + SET_ROWS(1, f16, _i64) - return vk_fa_pipeline_state{hsk, hsv, params.block_rows, params.block_cols, params.d_split, params.row_split, params.shmem_staging, params.path, params.workgroup_size, subgroup_size, aligned, f32acc, flags, params.limit_occupancy_shmem, k_type, v_type}; -} + // fork: turbo4_0 SET_ROWS uses a bespoke cooperative shader (set_rows_f32_turbo4_0.comp) + // with a self-contained WHT, so it is not part of the templated SET_ROWS macro above and + // exists for an f32 source only -- register it directly in the src_idx 0 (fp32) slot. + ggml_vk_create_pipeline(device, device->pipeline_set_rows_i32[0][GGML_TYPE_TURBO4_0], "set_rows_turbo4_0_i32", set_rows_turbo4_0_i32_len, set_rows_turbo4_0_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_set_rows_i64[0][GGML_TYPE_TURBO4_0], "set_rows_turbo4_0_i64", set_rows_turbo4_0_i64_len, set_rows_turbo4_0_i64_data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); +#undef SET_ROWS -// Bytes per buffer block for the FaBlockBytesK/V spec constants. F32 is fed as -// a vec4 "block" of 4 floats, everything else uses its ggml block size. -static uint32_t fa_block_bytes(ggml_type t) { - if (t == GGML_TYPE_F32) { - return 16u; - } - return (uint32_t) ggml_type_size(t); -} + // SP2 paged attention (GGML_OP_PAGED_ATTN_MT), F16 cache path (Task 3). + // Both shaders dispatch one workgroup per grid element (wg_denoms {1,1,1}), + // local_size_x = 128. scatter has 5 bindings, attention has 7. + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_F16], "paged_attn_scatter_f16", paged_attn_scatter_f16_len, paged_attn_scatter_f16_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_F16], "paged_attn_f16", paged_attn_f16_len, paged_attn_f16_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1, 1, 1}, {}, 1); -static std::vector<uint32_t> get_fa_spec_constants(const vk_fa_pipeline_state& state) { - return { - /* 0 WorkGroupSize */ state.workgroup_size, - /* 1 Br */ state.Br, - /* 2 Bc */ state.Bc, - /* 3 HSK */ state.HSK, - /* 4 HSV */ state.HSV, - /* 5 Clamp */ static_cast<uint32_t>(!state.aligned), - /* 6 D_split */ state.D_split, - /* 7 row_split */ state.row_split, - /* 8 SubGroupSize */ state.subgroup_size, - /* 9 SHMEM_STAGING */ state.shmem_staging ? 1u : 0u, - /*10 Flags */ state.flags, - /*11 LIMIT_OCCUPANCY_SHMEM */ state.limit_occupancy_shmem, - /*12 FaTypeK */ static_cast<uint32_t>(state.k_type), - /*13 FaTypeV */ static_cast<uint32_t>(state.v_type), - /*14 FaBlockBytesK */ fa_block_bytes(state.k_type), - /*15 FaBlockBytesV */ fa_block_bytes(state.v_type), - }; -} + // turbo4_0 cache path (Task 4): no-RHT cooperative scatter quantizer + dequant load. + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_TURBO4_0], "paged_attn_scatter_turbo4_0", paged_attn_scatter_turbo4_0_len, paged_attn_scatter_turbo4_0_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_TURBO4_0], "paged_attn_turbo4_0", paged_attn_turbo4_0_len, paged_attn_turbo4_0_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1, 1, 1}, {}, 1); -static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) { + // turbo4_64 cache path (Task 3 read side + Task 4 scatter): dequant load + // plus the no-RHT cooperative scatter quantizer (64-element blocks). + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_TURBO4_64], "paged_attn_scatter_turbo4_64", paged_attn_scatter_turbo4_64_len, paged_attn_scatter_turbo4_64_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_Q8_0], "paged_attn_scatter_q8_0", paged_attn_scatter_q8_0_len, paged_attn_scatter_q8_0_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_TURBO4_64], "paged_attn_turbo4_64", paged_attn_turbo4_64_len, paged_attn_turbo4_64_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_Q8_0], "paged_attn_q8_0", paged_attn_q8_0_len, paged_attn_q8_0_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1,1,1}, {}, 1); - uint32_t lut_size = 0; - switch (src0_type) { - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - // Regular matmul uses the compact uint16_t IQ1 grid; the expanded - // uint32_t grid is only enabled for the q8_1/int-dot vector path. - lut_size = 2*2048; - break; - case GGML_TYPE_IQ2_XXS: - lut_size = 8*256; - break; - case GGML_TYPE_IQ2_XS: - lut_size = 8*512; - break; - case GGML_TYPE_IQ2_S: - lut_size = 8*1024; - break; - case GGML_TYPE_IQ3_XXS: - lut_size = 4*256; - break; - case GGML_TYPE_IQ3_S: - lut_size = 4*512; - break; - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_MXFP4: - lut_size = 4*16; - break; - case GGML_TYPE_NVFP4: - // Same kvalues budget as MXFP4 plus ue4m3_fp32_lut[128] (types.glsl, DATA_A_NVFP4). - lut_size = 4*16 + 128u * (uint32_t)sizeof(float); - break; - default: - break; - } + // turbo4_64_ol cache path (SP2.5, 2026-07-01): turbo4_64 with 4 fixed-position + // outlier channels excluded from the group-norm/centroid quant, stored verbatim. + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_TURBO4_64_OL], "paged_attn_scatter_turbo4_64_ol", paged_attn_scatter_turbo4_64_ol_len, paged_attn_scatter_turbo4_64_ol_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_TURBO4_64_OL], "paged_attn_turbo4_64_ol", paged_attn_turbo4_64_ol_len, paged_attn_turbo4_64_ol_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1,1,1}, {}, 1); - // Needs to be kept up to date on shader changes - // Needs to stay aligned with ggml_vk_mul_mm_spec. - const bool intel_shmem_stride_pad_zero = device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && - device->driver_id == vk::DriverId::eIntelProprietaryWindows; - const uint32_t bank_conflict_offset = intel_shmem_stride_pad_zero ? 0 : (device->coopmat_support ? 8 : 1); - const uint32_t type_size = device->fp16 ? sizeof(ggml_fp16_t) : sizeof(float); - const uint32_t warps = warptile[0] / warptile[10]; + // turbo4_64_ol8 / turbo4_64_ol12 (2026-07-01 outlier-matrix sweep): same + // family as turbo4_64_ol above, just 8 and 12 fixed outlier channels + // respectively (see ggml-common.h TURBO4_64_OL8_OUTLIER_CHANNELS / + // TURBO4_64_OL12_OUTLIER_CHANNELS). Shader sources are shared with + // turbo4_64_ol via preprocessor defines (see vulkan-shaders-gen.cpp). + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_TURBO4_64_OL8], "paged_attn_scatter_turbo4_64_ol8", paged_attn_scatter_turbo4_64_ol8_len, paged_attn_scatter_turbo4_64_ol8_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_TURBO4_64_OL8], "paged_attn_turbo4_64_ol8", paged_attn_turbo4_64_ol8_len, paged_attn_turbo4_64_ol8_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_TURBO4_64_OL12], "paged_attn_scatter_turbo4_64_ol12", paged_attn_scatter_turbo4_64_ol12_len, paged_attn_scatter_turbo4_64_ol12_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_TURBO4_64_OL12], "paged_attn_turbo4_64_ol12", paged_attn_turbo4_64_ol12_len, paged_attn_turbo4_64_ol12_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1,1,1}, {}, 1); - const uint32_t load_bufs = (warptile[1] + warptile[2]) * (warptile[3] + bank_conflict_offset) * type_size; - const uint32_t mmid_row_ids = mul_mat_id ? (warptile[2] * 2 * sizeof(uint16_t)) : 0; - const uint32_t coopmat_stage = device->coopmat_support ? warptile[7] * warptile[8] / warps * sizeof(float) : 0; - const uint32_t ballots_sh = mul_mat_id ? (warps * 4 * sizeof(uint32_t)) : 0; + // SP2 Task 5: split-K decode. Pass-1 (decode) has 7 bindings per cache type; + // pass-2 (reduce) is type-agnostic with 3 bindings. Both wg_denoms {1,1,1}, local_size 128. + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_F16], "paged_attn_decode_f16", paged_attn_decode_f16_len, paged_attn_decode_f16_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_TURBO4_0], "paged_attn_decode_turbo4_0", paged_attn_decode_turbo4_0_len, paged_attn_decode_turbo4_0_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_TURBO4_64], "paged_attn_decode_turbo4_64", paged_attn_decode_turbo4_64_len, paged_attn_decode_turbo4_64_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_TURBO4_64_OL], "paged_attn_decode_turbo4_64_ol", paged_attn_decode_turbo4_64_ol_len, paged_attn_decode_turbo4_64_ol_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_TURBO4_64_OL8], "paged_attn_decode_turbo4_64_ol8", paged_attn_decode_turbo4_64_ol8_len, paged_attn_decode_turbo4_64_ol8_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_TURBO4_64_OL12], "paged_attn_decode_turbo4_64_ol12", paged_attn_decode_turbo4_64_ol12_len, paged_attn_decode_turbo4_64_ol12_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_Q8_0], "paged_attn_decode_q8_0", paged_attn_decode_q8_0_len, paged_attn_decode_q8_0_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1,1,1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode_reduce, "paged_attn_decode_reduce", paged_attn_decode_reduce_len, paged_attn_decode_reduce_data, "main", 3, sizeof(vk_op_paged_decode_reduce_pc), {1, 1, 1}, {}, 1); - const uint32_t total_size = load_bufs + mmid_row_ids + coopmat_stage + lut_size + ballots_sh; - const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; - VK_LOG_DEBUG("ggml_vk_matmul_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), " - "mul_mat_id=" << mul_mat_id << ", src0_type=" << ggml_type_name(src0_type) << ", supported=" << supported); + ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q1_0], "cpy_q1_0_f32", cpy_q1_0_f32_len, cpy_q1_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q1_0), 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q2_0], "cpy_q2_0_f32", cpy_q2_0_f32_len, cpy_q2_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q2_0), 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q4_0], "cpy_q4_0_f32", cpy_q4_0_f32_len, cpy_q4_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q4_0), 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q4_1], "cpy_q4_1_f32", cpy_q4_1_f32_len, cpy_q4_1_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q4_1), 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q5_0], "cpy_q5_0_f32", cpy_q5_0_f32_len, cpy_q5_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q5_0), 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q5_1], "cpy_q5_1_f32", cpy_q5_1_f32_len, cpy_q5_1_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q5_1), 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q8_0], "cpy_q8_0_f32", cpy_q8_0_f32_len, cpy_q8_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q8_0), 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_IQ4_NL], "cpy_iq4_nl_f32", cpy_iq4_nl_f32_len, cpy_iq4_nl_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_IQ4_NL), 1, 1}, {}, 1); - return supported; -} + auto get_suffix = [](bool src0_f16, bool src1_f16, bool dst_f16) { + std::string s; + s += std::string(src0_f16 ? "_f16" : "_f32"); + s += std::string(src1_f16 ? "_f16" : "_f32"); + s += std::string(dst_f16 ? "_f16" : "_f32"); + return s; + }; -// Shmem usage for the q8_1 mmq shader (mul_mmq.comp), which uses -// block_a_cache / block_b_cache layouts (see mul_mmq_shmem_types.glsl) rather -// than the float load buffers checked by ggml_vk_matmul_shmem_support. -// Sizes follow std430 rules. Returns false for types without a q8_1 pipeline. -static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) { +#define CREATE_BINARY(name, namemod, spec, bindings) \ + for (int s0 : {0,1}) for (int s1 : {0,1}) for (int d : {0,1}) \ + ggml_vk_create_pipeline2(device, device->pipeline_ ## name ## namemod[s0][s1][d], \ + #name + get_suffix(s0, s1, d) + #namemod, name ## _len[s0][s1][d], name ## _data[s0][s1][d], \ + "main", (bindings), sizeof(vk_op_binary_push_constants), {512, 1, 1}, spec, 1); - // FLOAT_TYPE in the shader is float16_t with fp16 support, otherwise float. - const uint32_t fp_size = device->fp16 ? 2u : 4u; - const uint32_t fp_align = fp_size; - const uint32_t fp2_size = 2u * fp_size; - const uint32_t fp2_align = device->fp16 ? 4u : 8u; + CREATE_BINARY(add, , {0}, 4) + CREATE_BINARY(add, _norepeat, {1}, 4) + CREATE_BINARY(sub, , {0}, 3) + CREATE_BINARY(sub, _norepeat, {1}, 3) + CREATE_BINARY(mul, , {0}, 3) + CREATE_BINARY(mul, _norepeat, {1}, 3) + CREATE_BINARY(div, , {0}, 3) + CREATE_BINARY(div, _norepeat, {1}, 3) + CREATE_BINARY(add_rms, , {0}, 4) + CREATE_BINARY(add_rms, _norepeat, {1}, 4) +#undef CREATE_BINARY - struct member { uint32_t size, align; }; - auto std430_size = [](std::initializer_list<member> members) { - uint32_t off = 0, struct_align = 1; - for (const auto &m : members) { - off = (off + m.align - 1) & ~(m.align - 1); - off += m.size; - struct_align = std::max(struct_align, m.align); + // f32 multi_add pipelines. f16 variants stay behind rte_fp16 (none today). + if (device->multi_add) { + for (uint32_t i = 0; i < MAX_FUSED_ADDS; ++i) { + ggml_vk_create_pipeline2(device, device->pipeline_multi_add[i], "multi_add_f32_" + std::to_string(i+1), multi_add_f32_len, multi_add_f32_data, "main", MAX_PARAMETER_COUNT, sizeof(vk_op_multi_add_push_constants), {512, 1, 1}, {i+2}, 1); + ggml_vk_create_pipeline2(device, device->pipeline_multi_add_rms[i], "multi_add_rms_f32_" + std::to_string(i+1), multi_add_rms_f32_len, multi_add_rms_f32_data, "main", MAX_PARAMETER_COUNT, sizeof(vk_op_multi_add_push_constants), {512, 1, 1}, {i+2}, 1); } - return (off + struct_align - 1) & ~(struct_align - 1); - }; - - uint32_t block_a_size = 0; - switch (src0_type) { - case GGML_TYPE_Q2_0: block_a_size = std430_size({{32, 4}, {fp_size, fp_align}}); break; // qs[8] + dm - case GGML_TYPE_Q4_0: block_a_size = std430_size({{16, 4}, {fp_size, fp_align}}); break; // qs[16/4] + dm - case GGML_TYPE_Q4_1: block_a_size = std430_size({{16, 4}, {fp2_size, fp2_align}}); break; // qs[16/4] + dm(vec2) - case GGML_TYPE_Q5_0: block_a_size = std430_size({{16, 4}, {4, 4}, {fp_size, fp_align}}); break; // qs[16/4] + qh + dm - case GGML_TYPE_Q5_1: block_a_size = std430_size({{16, 4}, {4, 4}, {fp2_size, fp2_align}}); break; // qs[16/4] + qh + dm(vec2) - case GGML_TYPE_Q8_0: block_a_size = std430_size({{32, 4}, {fp_size, fp_align}}); break; // qs[8] + dm - case GGML_TYPE_MXFP4: block_a_size = std430_size({{32, 4}, {fp_size, fp_align}}); break; // qs[8] + d - case GGML_TYPE_Q2_K: block_a_size = std430_size({{ 8, 4}, {2, 2}, {fp2_size, fp2_align}}); break; // qs[2] + scales(u8vec2) + dm(vec2) - case GGML_TYPE_Q3_K: block_a_size = std430_size({{16, 4}, {fp2_size, fp2_align}}); break; // qs[4] + d_scales(vec2) - case GGML_TYPE_Q4_K: block_a_size = std430_size({{16, 4}, {fp2_size, fp2_align}}); break; // qs[4] + dm(vec2) - case GGML_TYPE_Q5_K: block_a_size = std430_size({{32, 4}, {fp2_size, fp2_align}}); break; // qs[8] + dm(vec2) - case GGML_TYPE_Q6_K: block_a_size = std430_size({{32, 4}, {fp2_size, fp2_align}}); break; // qs[8] + d_scales(vec2) - default: - return false; } - // block_b_cache: { int32_t qs[8]; FLOAT_TYPEV2 ds; } - const uint32_t block_b_size = std430_size({{32, 4}, {fp2_size, fp2_align}}); - - const uint32_t BM = warptile[1]; - const uint32_t BN = warptile[2]; - // mul_mmq.comp: BK_STEP=1 for MUL_MAT_ID, 4 otherwise. - const uint32_t BK_STEP = mul_mat_id ? 1u : 4u; + ggml_vk_create_pipeline(device, device->pipeline_add_id_f32, "add_id_f32", add_id_f32_len, add_id_f32_data, "main", 4, sizeof(vk_op_add_id_push_constants), {1, 1, 1}, {}, 1); - const uint32_t buf_a_size = BM * BK_STEP * block_a_size; - const uint32_t buf_b_size = BN * BK_STEP * block_b_size; - const uint32_t mmid_row_ids = mul_mat_id ? (BN * 2u * (uint32_t)sizeof(uint16_t)) : 0u; + ggml_vk_create_pipeline(device, device->pipeline_acc_f32, "acc_f32", acc_f32_len, acc_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {0, 1}, 1); + ggml_vk_create_pipeline(device, device->pipeline_set_f32, "set_f32", acc_f32_len, acc_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {0, 0}, 1); - const uint32_t warps = warptile[0] / warptile[10]; - const uint32_t ballots_sh = mul_mat_id ? (warps * 4u * (uint32_t)sizeof(uint32_t)) : 0u; + ggml_vk_create_pipeline(device, device->pipeline_concat_i8, "concat_i8", concat_i8_len, concat_i8_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_concat_i16, "concat_i16", concat_i16_len, concat_i16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_concat_i32, "concat_i32", concat_i32_len, concat_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_concat_i64, "concat_i64", concat_i64_len, concat_i64_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + ballots_sh; - const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; + ggml_vk_create_pipeline(device, device->pipeline_upscale_nearest_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_NEAREST}, 1); + ggml_vk_create_pipeline(device, device->pipeline_upscale_bilinear_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BILINEAR}, 1); + ggml_vk_create_pipeline(device, device->pipeline_upscale_bicubic_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BICUBIC}, 1); + ggml_vk_create_pipeline(device, device->pipeline_upscale_bilinear_antialias_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS}, 1); - VK_LOG_DEBUG("ggml_vk_matmul_int_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), " - "mul_mat_id=" << mul_mat_id << ", src0_type=" << ggml_type_name(src0_type) << ", total=" << total_size << ", supported=" << supported); + ggml_vk_create_pipeline(device, device->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - return supported; -} + ggml_vk_create_pipeline(device, device->pipeline_log[0], "log_f32", log_f32_len, log_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_log[1], "log_f16", log_f16_len, log_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); -struct GpuPipelineConfig { - // GPU architecture identifier. - // Example: vk_device_architecture::AMD_GCN - vk_device_architecture arch; + ggml_vk_create_pipeline(device, device->pipeline_tri[0], "tri_f32", tri_f32_len, tri_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_tri[1], "tri_f16", tri_f16_len, tri_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - // Mapping of pipeline names to their specific subgroup sizes. - // Example: {"soft_max_f32", 64} - std::unordered_map<std::string, uint32_t> pipelines; + ggml_vk_create_pipeline(device, device->pipeline_diag[0], "diag_f32", diag_f32_len, diag_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_diag[1], "diag_f16", diag_f16_len, diag_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - // Default subgroup size for this GPU. - // Defaults to 0 if not explicitly provided. - uint32_t default_subgroup_size = 0; -}; + ggml_vk_create_pipeline(device, device->pipeline_pad_f32, "pad_f32", pad_f32_len, pad_f32_data, "main", 2, sizeof(vk_op_pad_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_pad_reflect_1d_f32, "pad_reflect_1d_f32", pad_reflect_1d_f32_len, pad_reflect_1d_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); -// Pipeline configuration for RDNA1 GPUs. -static const std::unordered_map<std::string, uint32_t> rdna1_pipelines = { - {"soft_max", 64}, {"im2col", 64}, - {"argmax", 64}, {"mul_mat_vec", 64}, - {"mul_mat_vec_f16", 32}, {"mul_mat_vec_f32_f16", 32} -}; + ggml_vk_create_pipeline(device, device->pipeline_roll_f32, "roll_f32", roll_f32_len, roll_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); -// Pipeline configuration for RDNA2 GPUs. -static const std::unordered_map<std::string, uint32_t> rdna2_pipelines = { - {"soft_max", 64}, {"im2col", 64}, -}; + ggml_vk_create_pipeline(device, device->pipeline_repeat_i32, "repeat_i32", repeat_i32_len, repeat_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_repeat_back_f32, "repeat_back_f32", repeat_back_f32_len, repeat_back_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); -static constexpr uint32_t RDNA_DEFAULT_SUBGROUP_SIZE = 32; + ggml_vk_create_pipeline(device, device->pipeline_repeat_i16, "repeat_i16", repeat_i16_len, repeat_i16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); -// Define configurations for different GPUs. -static std::vector<GpuPipelineConfig> gpu_pipeline_configs = { - { - vk_device_architecture::AMD_RDNA1, - { - rdna1_pipelines, - }, - RDNA_DEFAULT_SUBGROUP_SIZE - }, - { - vk_device_architecture::AMD_RDNA2, - { - rdna2_pipelines, - }, - RDNA_DEFAULT_SUBGROUP_SIZE - }, -}; +#define CREATE_UNARY(name) \ + ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \ + ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); -static uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch) { - for (const auto &config : gpu_pipeline_configs) { - if (config.arch == arch) { - auto pipIt = config.pipelines.find(pipeline_name); - if (pipIt != config.pipelines.end()) { - return pipIt->second; - } - std::vector<std::pair<std::string, uint32_t>> sorted_pipelines(config.pipelines.begin(), config.pipelines.end()); - std::sort(sorted_pipelines.begin(), sorted_pipelines.end(), - [](const auto &a, const auto &b) { return a.first.size() > b.first.size(); }); - for (const auto &entry : sorted_pipelines) { - if (pipeline_name.find(entry.first) != std::string::npos) { - return entry.second; - } - } - return config.default_subgroup_size; - } - } - return 0; // If no matching configuration is found -} - -// Whether scalar flash attention will use the MMQ path for the given K/V types. -static bool ggml_vk_fa_type_needs_shmem(ggml_type type) { - switch (type) { - case GGML_TYPE_IQ4_NL: - return true; - default: - return false; - } -} + CREATE_UNARY(elu) + CREATE_UNARY(gelu) + CREATE_UNARY(gelu_erf) + CREATE_UNARY(gelu_quick) + CREATE_UNARY(silu) + CREATE_UNARY(relu) + CREATE_UNARY(sqr) + CREATE_UNARY(sqrt) + CREATE_UNARY(sin) + CREATE_UNARY(cos) + CREATE_UNARY(clamp) + CREATE_UNARY(leaky_relu) + CREATE_UNARY(xielu) + CREATE_UNARY(neg) + CREATE_UNARY(tanh) + CREATE_UNARY(sigmoid) + CREATE_UNARY(hardsigmoid) + CREATE_UNARY(hardswish) + CREATE_UNARY(abs) + CREATE_UNARY(softplus) + CREATE_UNARY(step) + CREATE_UNARY(round) + CREATE_UNARY(ceil) + CREATE_UNARY(floor) + CREATE_UNARY(trunc) + CREATE_UNARY(sgn) + CREATE_UNARY(exp) + CREATE_UNARY(expm1) +#undef CREATE_UNARY -static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type, ggml_type v_type) { -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - return device->integer_dot_product && device->subgroup_clustered && - !ggml_vk_fa_type_needs_shmem(v_type) && - (k_type == GGML_TYPE_Q4_0 || k_type == GGML_TYPE_Q4_1 || - k_type == GGML_TYPE_Q5_0 || k_type == GGML_TYPE_Q5_1 || - k_type == GGML_TYPE_Q8_0); -#else - GGML_UNUSED(device); - GGML_UNUSED(k_type); - GGML_UNUSED(v_type); - return false; -#endif -} +// spec constants: {norepeat, op_on_b} +#define CREATE_UNARY_MUL(name, idx) \ + for (int dt = 0; dt < 2; ++dt) { \ + const size_t len_ = dt ? name ## _mul_f16_len : name ## _mul_f32_len; \ + const unsigned char * data_ = dt ? name ## _mul_f16_data : name ## _mul_f32_data; \ + const std::string dts_ = dt ? "f16" : "f32"; \ + for (int ob = 0; ob < 2; ++ob) \ + for (int nr = 0; nr < 2; ++nr) \ + ggml_vk_create_pipeline(device, device->pipeline_unary_mul[(idx)][dt][nr][ob], \ + (#name "_mul" + std::string(ob ? "_b" : "") + "_" + dts_ + (nr ? "_norepeat" : "")).c_str(), \ + len_, data_, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, \ + { (uint32_t) nr, (uint32_t) ob }, 1); \ + } + + CREATE_UNARY_MUL(gelu, 0) + CREATE_UNARY_MUL(sigmoid, 1) + CREATE_UNARY_MUL(silu, 2) + CREATE_UNARY_MUL(softplus, 3) +#undef CREATE_UNARY_MUL -// load_shaders walks the pipeline list under compile_mutex and either claims -// the requested pipeline for compilation or, if another thread is already -// compiling it, drops the lock and waits on compile_cv. Compiles themselves -// run unlocked. -struct CompileTask { - vk_pipeline pipeline; - size_t spv_size; - const void * spv_data; - std::string entrypoint; - uint32_t parameter_count; - std::array<uint32_t, 3> wg_denoms; - std::vector<uint32_t> specialization_constants; - bool disable_robustness; - bool require_full_subgroups; - uint32_t required_subgroup_size; -}; + ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f16, "add1_f16_f16", add1_f16_f16_len, add1_f16_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f32, "add1_f16_f32", add1_f16_f32_len, add1_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_add1_f32_f32, "add1_f32_f32", add1_f32_f32_len, add1_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); -static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { - VK_LOG_DEBUG("ggml_vk_load_shaders(" << device->name << ")"); + ggml_vk_create_pipeline(device, device->pipeline_arange_f32, "arange_f32", arange_f32_len, arange_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - // some shaders have a minimum subgroup size - const uint32_t subgroup_size_8 = std::max(device->subgroup_size, 8u); - const uint32_t subgroup_size_16 = std::max(device->subgroup_size, 16u); - const uint32_t subgroup_size_32 = std::max(device->subgroup_size, 32u); + ggml_vk_create_pipeline(device, device->pipeline_fill_f32, "fill_f32", fill_f32_len, fill_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_fill_f16, "fill_f16", fill_f16_len, fill_f16_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - // clamp WARP for l_/m_ warptiles so WM <= BM (breaks on subgroupSize > 64) - const uint32_t mm_warp_8 = std::min(subgroup_size_8, 64u); - const uint32_t mm_warp_16 = std::min(subgroup_size_16, 64u); +#define CREATE_GLU(name) \ + ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); - const uint32_t mul_mat_subgroup_size = (device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control) ? device->subgroup_min_size : device->subgroup_size; - const uint32_t mul_mat_subgroup_size_8 = std::max(mul_mat_subgroup_size, 8u); - const uint32_t mul_mat_subgroup_size_16 = std::max(mul_mat_subgroup_size, 16u); - const uint32_t mul_mat_subgroup_size_32 = std::max(mul_mat_subgroup_size, 32u); - const uint32_t mul_mat_mm_warp_8 = std::min(mul_mat_subgroup_size_8, 64u); - const uint32_t mul_mat_mm_warp_16 = std::min(mul_mat_subgroup_size_16, 64u); + CREATE_GLU(geglu) + CREATE_GLU(reglu) + CREATE_GLU(swiglu) + CREATE_GLU(swiglu_oai) + CREATE_GLU(swiglu_clamp) + CREATE_GLU(geglu_erf) + CREATE_GLU(geglu_quick) +#undef CREATE_GLU - const bool subgroup_min_size_16 = (!device->subgroup_size_control && device->subgroup_size >= 16) || - (device->subgroup_size_control && device->subgroup_max_size >= 16); + ggml_vk_create_pipeline(device, device->pipeline_silu_back_f32, "silu_back_f32", silu_back_f32_len, silu_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - // mulmat - std::vector<uint32_t> l_warptile, m_warptile, s_warptile, - l_warptile_id, m_warptile_id, s_warptile_id, - l_warptile_mmq, m_warptile_mmq, s_warptile_mmq, - l_warptile_mmq_int, m_warptile_mmq_int, s_warptile_mmq_int, - l_warptile_mmq_int_k, m_warptile_mmq_int_k, s_warptile_mmq_int_k, - l_warptile_mmq_k, m_warptile_mmq_k, s_warptile_mmq_k, - l_warptile_mmqid, m_warptile_mmqid, s_warptile_mmqid, - l_warptile_mmqid_int, m_warptile_mmqid_int, s_warptile_mmqid_int, - l_warptile_mmqid_int_k, m_warptile_mmqid_int_k, s_warptile_mmqid_int_k; - std::array<uint32_t, 3> l_wg_denoms, m_wg_denoms, s_wg_denoms, - l_mmq_wg_denoms, m_mmq_wg_denoms, s_mmq_wg_denoms, - l_mmq_wg_denoms_k, m_mmq_wg_denoms_k, s_mmq_wg_denoms_k, - l_mmqid_wg_denoms, m_mmqid_wg_denoms, s_mmqid_wg_denoms; + ggml_vk_create_pipeline(device, device->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {1, 512, 1}, {}, 1, true); - uint32_t l_align, m_align, s_align; + ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32, "soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_wg512, "soft_max_f32_wg512", soft_max_f32_len, soft_max_f32_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 512 }, 1); + ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_f16, "soft_max_f32_f16", soft_max_f32_f16_len, soft_max_f32_f16_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_f16_wg512, "soft_max_f32_f16_wg512", soft_max_f32_f16_len, soft_max_f32_f16_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 512 }, 1); + ggml_vk_create_pipeline(device, device->pipeline_soft_max_back_f32, "soft_max_back_f32", soft_max_back_f32_len, soft_max_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1, true); - vk_pipeline wait_pipeline; - CompileTask claimed_task {}; - bool has_claimed_task = false; + ggml_vk_create_pipeline(device, device->pipeline_soft_max_large1_f32, "soft_max_large1_f32", soft_max_large1_f32_len, soft_max_large1_f32_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_soft_max_large2_f32, "soft_max_large2_f32", soft_max_large2_f32_len, soft_max_large2_f32_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_soft_max_large3_f32, "soft_max_large3_f32", soft_max_large3_f32_len, soft_max_large3_f32_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_soft_max_large1_f32_f16, "soft_max_large1_f32_f16", soft_max_large1_f32_f16_len, soft_max_large1_f32_f16_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_soft_max_large2_f32_f16, "soft_max_large2_f32_f16", soft_max_large2_f32_f16_len, soft_max_large2_f32_f16_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_soft_max_large3_f32_f16, "soft_max_large3_f32_f16", soft_max_large3_f32_f16_len, soft_max_large3_f32_f16_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); - // The rest of the walk reads and writes shared device state, so hold the - // lock until we're done deciding what to compile. - std::unique_lock<std::mutex> compile_lock(device->compile_mutex); + ggml_vk_create_pipeline(device, device->pipeline_rope_norm_f32, "rope_norm_f32", rope_norm_f32_len, rope_norm_f32_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_rope_neox_f32, "rope_neox_f32", rope_neox_f32_len, rope_neox_f32_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_rope_multi_f32, "rope_multi_f32", rope_multi_f32_len, rope_multi_f32_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_rope_vision_f32, "rope_vision_f32", rope_vision_f32_len, rope_vision_f32_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - if (device->coopmat2) { - // spec constants and tile sizes for non-quant matmul/matmul_id - l_warptile = { 256, 128, 256, 64, 1 }; - m_warptile = { 256, 128, 128, 64, 0 }; - s_warptile = { 128, 64, 64, 64, 0 }; - l_wg_denoms = {128, 256, 1 }; - m_wg_denoms = {128, 128, 1 }; - s_wg_denoms = { 64, 64, 1 }; + ggml_vk_create_pipeline(device, device->pipeline_rope_norm_f16, "rope_norm_f16", rope_norm_f16_len, rope_norm_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_rope_neox_f16, "rope_neox_f16", rope_neox_f16_len, rope_neox_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_rope_multi_f16, "rope_multi_f16", rope_multi_f16_len, rope_multi_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_rope_vision_f16, "rope_vision_f16", rope_vision_f16_len, rope_vision_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - // spec constants and tile sizes for quant matmul (non-Qi_K) - l_warptile_mmq = { 256, 128, 256, 64, 1 }; - m_warptile_mmq = { 256, 128, 128, 64, 1 }; - s_warptile_mmq = { 256, 32, 64, 128, 0 }; - l_mmq_wg_denoms = { 128, 256, 1 }; - m_mmq_wg_denoms = { 128, 128, 1 }; - s_mmq_wg_denoms = { 32, 64, 1 }; + ggml_vk_create_pipeline(device, device->pipeline_rope_norm_f32_f16, "rope_norm_f32_f16", rope_norm_f32_f16_len, rope_norm_f32_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_rope_neox_f32_f16, "rope_neox_f32_f16", rope_neox_f32_f16_len, rope_neox_f32_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_rope_multi_f32_f16, "rope_multi_f32_f16", rope_multi_f32_f16_len, rope_multi_f32_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - // spec constants and tile sizes for quant matmul (Qi_K) - l_warptile_mmq_k = { 256, 128, 256, 64, 1 }; - m_warptile_mmq_k = { 256, 128, 128, 64, 1 }; - s_warptile_mmq_k = { 256, 32, 64, 128, 0 }; - l_mmq_wg_denoms_k = { 128, 256, 1 }; - m_mmq_wg_denoms_k = { 128, 128, 1 }; - s_mmq_wg_denoms_k = { 32, 64, 1 }; + for (uint32_t i = 0; i < num_argsort_pipelines; ++i) { + uint32_t BLOCK_SIZE = 1u << std::min(i, device->max_workgroup_size_log2); + if (i <= device->max_workgroup_size_log2 && + 2 * sizeof(int) * BLOCK_SIZE <= device->properties.limits.maxComputeSharedMemorySize) { + const uint32_t NCOLS_PADDED_LOG2 = i; + ggml_vk_create_pipeline2(device, device->pipeline_argsort_f32[i], "argsort_f32_"+std::to_string(i), argsort_f32_len, argsort_f32_data, "main", 3, sizeof(vk_op_argsort_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, NCOLS_PADDED_LOG2}, 1, true); + } + const uint32_t WG_UNROLL_FACTOR = BLOCK_SIZE > 1 ? 2 : 1; + BLOCK_SIZE /= WG_UNROLL_FACTOR; + ggml_vk_create_pipeline2(device, device->pipeline_argsort_large_f32[i], "argsort_large_f32_"+std::to_string(i), argsort_large_f32_len, argsort_large_f32_data, "main", 3, sizeof(vk_op_argsort_push_constants), {BLOCK_SIZE * WG_UNROLL_FACTOR, 1, 1}, {BLOCK_SIZE, WG_UNROLL_FACTOR}, 1, true); + } - // spec constants and tile sizes for quant matmul_id - const uint32_t mmqid_bk = device->coopmat2_decode_vector ? 64u : 32u; - l_warptile_mmqid = { 256, 128, 128, mmqid_bk, 1, device->subgroup_size }; - m_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0, device->subgroup_size }; - s_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0, device->subgroup_size }; - l_mmqid_wg_denoms = { 128, 128, 1 }; - m_mmqid_wg_denoms = { 128, 64, 1 }; - s_mmqid_wg_denoms = { 128, 64, 1 }; + for (uint32_t i = 0; i < num_topk_pipelines; ++i) { + const uint32_t BLOCK_SIZE = 1u << i; + const uint32_t NCOLS_PADDED_LOG2 = i; + if (i <= device->max_workgroup_size_log2) { + uint32_t nary_shmem = 2 * sizeof(int) * BLOCK_SIZE + + sizeof(int) * device->subgroup_size + + 2 * sizeof(int) + + 2 * (BLOCK_SIZE / device->subgroup_size) * sizeof(int); + if (device->subgroup_arithmetic && device->subgroup_require_full_support && device->subgroup_shuffle && device->subgroup_ballot && + nary_shmem <= device->properties.limits.maxComputeSharedMemorySize) { + ggml_vk_create_pipeline2(device, device->pipeline_topk_f32[i], "topk_f32_"+std::to_string(i), topk_nary_search_f32_len, topk_nary_search_f32_data, "main", 2, sizeof(vk_op_topk_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, device->subgroup_size, device->subgroup_size_log2}, 1, true, true, device->subgroup_size); + } else if (2 * sizeof(int) * BLOCK_SIZE <= device->properties.limits.maxComputeSharedMemorySize) { + ggml_vk_create_pipeline2(device, device->pipeline_topk_f32[i], "topk_f32_"+std::to_string(i), topk_argsort_f32_len, topk_argsort_f32_data, "main", 2, sizeof(vk_op_topk_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, NCOLS_PADDED_LOG2}, 1, true); + } + } + } - l_align = 128; - m_align = 64; - s_align = 32; - } else { - // Matrix cores require different warp group sizes - const uint32_t tm_l = device->coopmat_support ? device->coopmat_m : 4; - const uint32_t tm_m = device->coopmat_support ? device->coopmat_m : 4; - const uint32_t tm_s = device->coopmat_support ? device->coopmat_m : 2; - const uint32_t tn_l = device->coopmat_support ? device->coopmat_n : 4; - const uint32_t tn_m = device->coopmat_support ? device->coopmat_n : 2; - const uint32_t tn_s = device->coopmat_support ? device->coopmat_n : 2; - const uint32_t tk_l = device->coopmat_support ? device->coopmat_k : 1; - const uint32_t tk_m = device->coopmat_support ? device->coopmat_k : 1; - const uint32_t tk_s = device->coopmat_support ? device->coopmat_k : 1; + // large-k fallback: one workgroup per row, radix-select instead of a full sort. The QSA + // variant (spec constant 1) additionally gathers the qwen4 indexer input on the fly. + { + const uint32_t BLOCK_SIZE = 1u << std::min(10u, device->max_workgroup_size_log2); + ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_f32, "topk_radix_f32", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 0}, 1, true); + ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_qsa, "topk_radix_qsa", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 1}, 1, true); + } - const uint32_t s_warptile_wm = device->subgroup_size == 8 ? 8 : 32; + ggml_vk_create_pipeline(device, device->pipeline_argmax_f32, "argmax_f32", argmax_f32_len, argmax_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); - l_warptile = { 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, tm_l, tn_l, tk_l, mm_warp_8 }; - m_warptile = { 128, 64, 64, 16, mm_warp_8, 32, 2, tm_m, tn_m, tk_m, mm_warp_8 }; - s_warptile = { subgroup_size_32, 32, 32, 16, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, subgroup_size_8 }; + ggml_vk_create_pipeline(device, device->pipeline_sum_rows_f32, "sum_rows_f32", sum_rows_f32_len, sum_rows_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_f32, "cross_entropy_loss_f32", cross_entropy_loss_f32_len, cross_entropy_loss_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_f32_wg512, "cross_entropy_loss_f32_wg512", cross_entropy_loss_f32_len, cross_entropy_loss_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { 512 }, 1); + ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_back_f32, "cross_entropy_loss_back_f32", cross_entropy_loss_back_f32_len, cross_entropy_loss_back_f32_data, "main", 4, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_back_f32_wg512, "cross_entropy_loss_back_f32_wg512", cross_entropy_loss_back_f32_len, cross_entropy_loss_back_f32_data, "main", 4, sizeof(vk_op_push_constants), {1, 1, 1}, { 512 }, 1); + // Intel Windows driver in range [32.0.101.8509, 32.0.101.8860) will crash when using fwht kernels so we gate that here + const bool can_use_fwht = device->driver_id != vk::DriverId::eIntelProprietaryWindows || + !ggml_vk_intel_windows_driver_in_range(device->properties.driverVersion, 101, 8509, 101, 8860); + if (can_use_fwht && device->subgroup_basic && device->subgroup_shuffle) { + int idx = 0; + for (uint32_t n : {64, 128, 256, 512}) { + if (device->subgroup_size <= n) { + ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_f32", fwht_f32_len, fwht_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { device->subgroup_size, n }, 1, true, true, device->subgroup_size); + } + ++idx; + } + } else if (can_use_fwht) { + int idx = 0; + for (uint32_t n : {64, 128, 256, 512}) { + const uint32_t block_size = std::min(device->subgroup_size, n); + ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_shmem_f32", fwht_shmem_f32_len, fwht_shmem_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { block_size, n }, 1); + ++idx; + } + } - l_warptile_mmq = { 128, 128, 128, 32, mm_warp_8 * 2, 64, 2, tm_l, tn_l, tk_l, mm_warp_8 }; - m_warptile_mmq = { 128, 64, 64, 32, mm_warp_8, 32, 2, tm_m, tn_m, tk_m, mm_warp_8 }; - s_warptile_mmq = { subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, subgroup_size_8 }; + const uint32_t cumsum_elem_per_thread = (device->vendor_id == VK_VENDOR_ID_AMD || device->vendor_id == VK_VENDOR_ID_INTEL) ? 2 : 4; + ggml_vk_create_pipeline(device, device->pipeline_cumsum_f32, "cumsum_f32", cumsum_f32_len, cumsum_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { 256, device->subgroup_size, cumsum_elem_per_thread }, 1, true, true, device->subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_cumsum_small_f32, "cumsum_f32", cumsum_f32_len, cumsum_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { 128, device->subgroup_size, 1 }, 1, true, true, device->subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_cumsum_multipass1_f32, "cumsum_multipass1_f32", cumsum_multipass1_f32_len, cumsum_multipass1_f32_data, "main", 3, sizeof(vk_op_sum_rows_push_constants), {256, 1, 1}, { 256, device->subgroup_size }, 1, true, true, device->subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_cumsum_multipass2_f32, "cumsum_multipass2_f32", cumsum_multipass2_f32_len, cumsum_multipass2_f32_data, "main", 3, sizeof(vk_op_sum_rows_push_constants), {256, 1, 1}, { 256, device->subgroup_size }, 1, true, true, device->subgroup_size); - // Integer MMQ has a smaller shared memory profile, but heavier register use - l_warptile_mmq_int = { 128, 128, 128, 32, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 }; - m_warptile_mmq_int = { 128, 64, 64, 32, mm_warp_8, 32, 2, 2, 2, 1, mm_warp_8 }; - s_warptile_mmq_int = { subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, 2, 1, 1, subgroup_size_8 }; + ggml_vk_create_pipeline(device, device->pipeline_count_equal_i32, "count_equal_i32", count_equal_i32_len, count_equal_i32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, { device->subgroup_size }, 1); - // K-quants use even more registers, mitigate by setting WMITER to 1 - l_warptile_mmq_int_k = { 128, 128, 128, 32, mm_warp_8 * 2, 64, 1, 4, 4, 1, mm_warp_8 }; - m_warptile_mmq_int_k = { 128, 64, 64, 32, mm_warp_8, 32, 1, 2, 2, 1, mm_warp_8 }; - s_warptile_mmq_int_k = { subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 1, 2, 1, 1, subgroup_size_8 }; + if (device->subgroup_arithmetic && device->subgroup_require_full_support) { + ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_subgroup_len, count_experts_subgroup_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true, true); + } else { + ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_len, count_experts_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true); + } - l_warptile_id = { 128, 128, 128, 16, mul_mat_mm_warp_16 * 2, 64, 2, tm_l, tn_l, tk_l, mul_mat_mm_warp_16 }; - m_warptile_id = { 128, 64, 64, 16, mul_mat_mm_warp_16, 32, 2, tm_m, tn_m, tk_m, mul_mat_mm_warp_16 }; - s_warptile_id = { mul_mat_subgroup_size_16, 32, 32, 16, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, mul_mat_subgroup_size_16 }; + // comb holds a token's 4x4 matrix in one 16-lane slice of a subgroup, so it + // needs at least 16 lanes, pinned to a known size. + if (device->subgroup_basic && device->subgroup_shuffle && device->subgroup_require_full_support && device->subgroup_size >= 16) { + const uint32_t tokens_per_workgroup = 4 * (device->subgroup_size / 16); + ggml_vk_create_pipeline(device, device->pipeline_dsv4_hc_comb_f32, "dsv4_hc_comb_f32", dsv4_hc_comb_f32_len, dsv4_hc_comb_f32_data, "main", 4, sizeof(vk_op_dsv4_hc_comb_push_constants), {tokens_per_workgroup, 1, 1}, { device->subgroup_size }, 1, true, true, device->subgroup_size); + } - l_warptile_mmqid = { 128, 128, 128, 32, mul_mat_mm_warp_8 * 2, 64, 2, tm_l, tn_l, tk_l, mul_mat_mm_warp_8 }; - m_warptile_mmqid = { 128, 64, 64, 32, mul_mat_mm_warp_8, 32, 2, tm_m, tn_m, tk_m, mul_mat_mm_warp_8 }; - s_warptile_mmqid = { mul_mat_subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, tm_s, tn_s, tk_s, mul_mat_subgroup_size_8 }; + ggml_vk_create_pipeline(device, device->pipeline_dsv4_hc_pre_f32, "dsv4_hc_pre_f32", dsv4_hc_pre_f32_len, dsv4_hc_pre_f32_data, "main", 3, sizeof(vk_op_dsv4_hc_pre_push_constants), {256, 1, 1}, { 256, 0 }, 1); + ggml_vk_create_pipeline(device, device->pipeline_dsv4_hc_pre_gated_f32, "dsv4_hc_pre_gated_f32", dsv4_hc_pre_f32_len, dsv4_hc_pre_f32_data, "main", 3, sizeof(vk_op_dsv4_hc_pre_push_constants), {256, 1, 1}, { 256, 1 }, 1); + ggml_vk_create_pipeline(device, device->pipeline_dsv4_hc_post_f32, "dsv4_hc_post_f32", dsv4_hc_post_f32_len, dsv4_hc_post_f32_data, "main", 5, sizeof(vk_op_dsv4_hc_post_push_constants), {256, 1, 1}, { 256, 1 }, 1); + ggml_vk_create_pipeline(device, device->pipeline_dsv4_hc_post_nocomb_f32,"dsv4_hc_post_nocomb_f32",dsv4_hc_post_f32_len, dsv4_hc_post_f32_data, "main", 5, sizeof(vk_op_dsv4_hc_post_push_constants), {256, 1, 1}, { 256, 0 }, 1); - l_warptile_mmqid_int = { 128, 128, 128, 32, mul_mat_mm_warp_8 * 2, 64, 2, 4, 4, 1, mul_mat_mm_warp_8 }; - m_warptile_mmqid_int = { 128, 64, 64, 32, mul_mat_mm_warp_8, 32, 2, 2, 2, 1, mul_mat_mm_warp_8 }; - s_warptile_mmqid_int = { mul_mat_subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 2, 2, 1, 1, mul_mat_subgroup_size_8 }; + for (auto &s : device->pipeline_solve_tri_f32) { + const vk_solve_tri_pipeline_state &state = s.first; - l_warptile_mmqid_int_k = { 128, 128, 128, 32, mul_mat_mm_warp_16 * 2, 64, 1, 4, 4, 1, mul_mat_mm_warp_16 }; - m_warptile_mmqid_int_k = { 128, 64, 64, 32, mul_mat_mm_warp_16, 32, 1, 2, 2, 1, mul_mat_mm_warp_16 }; - s_warptile_mmqid_int_k = { mul_mat_subgroup_size_32, 32, 32, 32, s_warptile_wm, 32, 1, 2, 1, 1, mul_mat_subgroup_size_16 }; + // Max number of rows to load at a time, limited by shared memory + const uint32_t batch_N = device->properties.limits.maxComputeSharedMemorySize / ((state.N + state.K) * sizeof(float)); + // Need at least K invocations, and prefer a minimum of 128 to spread out loading shared memory + const uint32_t block_size = std::max(128u, 1u << (uint32_t)ceilf(log2f(float(state.K)))); - // chip specific tuning - if ((device->architecture == AMD_GCN) && (device->driver_id != vk::DriverId::eAmdProprietary)) { - m_warptile_mmq = m_warptile_mmq_int = { 256, 64, 64, 32, 16, 16, 2, 2, 2, 1, 16 }; - m_warptile_mmqid = m_warptile_mmqid_int = { 256, 64, 64, 32, 16, 16, 2, 2, 2, 1, 16 }; - } else if (device->vendor_id == VK_VENDOR_ID_AMD && device->coopmat_support && device->driver_id != vk::DriverId::eAmdProprietary) { - // This is intentionally using tx_m values, slight performance increase - l_warptile = { 256, 128, 128, 16, mm_warp_8, 64, 2, tm_m, tn_m, tk_m, mm_warp_8 }; - l_warptile_mmq = l_warptile_mmq_int = { 256, 128, 128, 32, mm_warp_8, 64, 2, tm_m, tn_m, tk_m, mm_warp_8 }; - l_warptile_mmq_int_k = { 256, 128, 128, 32, mm_warp_16, 64, 1, 4, 2, 1, mm_warp_16 }; - } else if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support) { - // Xe2/Xe3 with coopmat enabled - warptile performance tuning - l_warptile = { 512, 128, 128, 16, mm_warp_8, 32, 2, tm_m, tn_m, tk_m, mm_warp_8 }; - l_warptile_mmq = { 512, 128, 128, 32, mm_warp_8, 32, 2, tm_m, tn_m, tk_m, mm_warp_8 }; - } + ggml_vk_create_pipeline( + device, s.second, "solve_tri_f32", + solve_tri_f32_len, solve_tri_f32_data, "main", 3, + sizeof(vk_op_binary_push_constants), {1, 1, 1}, { 0, state.N, state.K, batch_N, block_size }, 1, true); + } - l_mmq_wg_denoms = l_wg_denoms = {128, 128, 1 }; - m_mmq_wg_denoms = m_wg_denoms = { 64, 64, 1 }; - s_mmq_wg_denoms = s_wg_denoms = { 32, 32, 1 }; - l_align = 128; - m_align = 64; - s_align = 32; +#define IM2COL(bda) \ + ggml_vk_create_pipeline(device, device->pipeline_im2col_f32, "im2col_f32", im2col_f32 ## bda ## _len, im2col_f32 ## bda ## _data, "main", 2, sizeof(vk_op_im2col_push_constants), {512, 1, 1}, { device->subgroup_size }, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_im2col_3d_f32, "im2col_3d_f32", im2col_3d_f32 ## bda ## _len, im2col_3d_f32 ## bda ## _data, "main", 2, sizeof(vk_op_im2col_3d_push_constants), {512, 1, 1}, { 512 }, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_im2col_f32_f16, "im2col_f32_f16", im2col_f32_f16 ## bda ## _len, im2col_f32_f16 ## bda ## _data, "main", 2, sizeof(vk_op_im2col_push_constants), {512, 1, 1}, { device->subgroup_size }, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_im2col_3d_f32_f16, "im2col_3d_f32_f16", im2col_3d_f32_f16 ## bda ## _len, im2col_3d_f32_f16 ## bda ## _data, "main", 2, sizeof(vk_op_im2col_3d_push_constants), {512, 1, 1}, { 512 }, 1, true); + if (device->shader_int64 && device->buffer_device_address) { + IM2COL(_bda) + } else { + IM2COL() + } - for (uint32_t i = 0; i < GGML_TYPE_COUNT; ++i) { - ggml_type t = (ggml_type)i; - // Disable medium and large matrix multiplication if not enough shared memory is available - // Check mmq warptiles as the largest configuration - // Throw an error if not enough for any matrix multiplication is available - if (!ggml_vk_matmul_shmem_support(device, s_warptile_mmq, false, t)) { - std::cerr << "ggml_vulkan: Error: Shared memory size too small for matrix multiplication." << std::endl; - throw std::runtime_error("Shared memory size too small for matrix multiplication."); - } else if (!ggml_vk_matmul_shmem_support(device, m_warptile_mmq, false, t)) { - device->mul_mat_m[i] = false; - device->mul_mat_l[i] = false; - } else if (!ggml_vk_matmul_shmem_support(device, l_warptile_mmq, false, t)) { - device->mul_mat_l[i] = false; - } + ggml_vk_create_pipeline(device, device->pipeline_timestep_embedding_f32, "timestep_embedding_f32", timestep_embedding_f32_len, timestep_embedding_f32_data, "main", 2, sizeof(vk_op_timestep_embedding_push_constants), {256, 1, 1}, {}, 1); - // Disable mul_mat_id if not enough shared memory is available - if (!ggml_vk_matmul_shmem_support(device, s_warptile_mmqid, true, t)) { - device->mul_mat_id_s[i] = false; - device->mul_mat_id_m[i] = false; - device->mul_mat_id_l[i] = false; - } else if (!ggml_vk_matmul_shmem_support(device, m_warptile_mmqid, true, t)) { - device->mul_mat_id_m[i] = false; - device->mul_mat_id_l[i] = false; - } else if (!ggml_vk_matmul_shmem_support(device, l_warptile_mmqid, true, t)) { - device->mul_mat_id_l[i] = false; - } + ggml_vk_create_pipeline(device, device->pipeline_conv_transpose_1d_f32, "conv_transpose_1d_f32", conv_transpose_1d_f32_len, conv_transpose_1d_f32_data, "main", 3, sizeof(vk_op_conv_transpose_1d_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_col2im_1d_f32, "col2im_1d_f32", col2im_1d_f32_len, col2im_1d_f32_data, "main", 2, sizeof(vk_op_col2im_1d_push_constants), {256, 1, 1}, {}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_col2im_1d_f16, "col2im_1d_f16", col2im_1d_f16_len, col2im_1d_f16_data, "main", 2, sizeof(vk_op_col2im_1d_push_constants), {256, 1, 1}, {}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_col2im_1d_bf16, "col2im_1d_bf16", col2im_1d_bf16_len, col2im_1d_bf16_data, "main", 2, sizeof(vk_op_col2im_1d_push_constants), {256, 1, 1}, {}, 1, true); - // The q8_1 mmq path has its own (larger) shmem layout, check it separately. - // K-quants use the _int_k warptiles, others use _int. - const bool is_k_quant = (t == GGML_TYPE_Q2_K || t == GGML_TYPE_Q3_K || - t == GGML_TYPE_Q4_K || t == GGML_TYPE_Q5_K || - t == GGML_TYPE_Q6_K); - const auto & s_int = is_k_quant ? s_warptile_mmq_int_k : s_warptile_mmq_int; - const auto & m_int = is_k_quant ? m_warptile_mmq_int_k : m_warptile_mmq_int; - const auto & l_int = is_k_quant ? l_warptile_mmq_int_k : l_warptile_mmq_int; - const auto & s_intid = is_k_quant ? s_warptile_mmqid_int_k : s_warptile_mmqid_int; - const auto & m_intid = is_k_quant ? m_warptile_mmqid_int_k : m_warptile_mmqid_int; - const auto & l_intid = is_k_quant ? l_warptile_mmqid_int_k : l_warptile_mmqid_int; + ggml_vk_create_pipeline(device, device->pipeline_out_prod_f32, "out_prod_f32", out_prod_f32_len, out_prod_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {256, 1, 1}, {}, 1); - if (!ggml_vk_matmul_int_shmem_support(device, s_int, false, t)) { - device->mul_mat_s_int[i] = false; - device->mul_mat_m_int[i] = false; - device->mul_mat_l_int[i] = false; - } else if (!ggml_vk_matmul_int_shmem_support(device, m_int, false, t)) { - device->mul_mat_m_int[i] = false; - device->mul_mat_l_int[i] = false; - } else if (!ggml_vk_matmul_int_shmem_support(device, l_int, false, t)) { - device->mul_mat_l_int[i] = false; - } + ggml_vk_create_pipeline(device, device->pipeline_snake_f32, "snake_f32", snake_f32_len, snake_f32_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_snake_f16, "snake_f16", snake_f16_len, snake_f16_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_snake_bf16, "snake_bf16", snake_bf16_len, snake_bf16_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1); - if (!ggml_vk_matmul_int_shmem_support(device, s_intid, true, t)) { - device->mul_mat_id_s_int[i] = false; - device->mul_mat_id_m_int[i] = false; - device->mul_mat_id_l_int[i] = false; - } else if (!ggml_vk_matmul_int_shmem_support(device, m_intid, true, t)) { - device->mul_mat_id_m_int[i] = false; - device->mul_mat_id_l_int[i] = false; - } else if (!ggml_vk_matmul_int_shmem_support(device, l_intid, true, t)) { - device->mul_mat_id_l_int[i] = false; - } - } - } + ggml_vk_create_pipeline(device, device->pipeline_pool1d_f32, "pool1d_f32", pool1d_f32_len, pool1d_f32_data, "main", 2, sizeof(vk_op_pool1d_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_pool2d_f32, "pool2d_f32", pool2d_f32_len, pool2d_f32_data, "main", 2, sizeof(vk_op_pool2d_push_constants), {512, 1, 1}, {}, 1); - if (!device->pipeline_matmul_f32) { - device->pipeline_matmul_f32 = std::make_shared<vk_matmul_pipeline_struct>(); - } - if (!device->pipeline_matmul_f32_f16) { - device->pipeline_matmul_f32_f16 = std::make_shared<vk_matmul_pipeline_struct>(); - } - if (!device->pipeline_matmul_id_f32) { - device->pipeline_matmul_id_f32 = std::make_shared<vk_matmul_pipeline_struct>(); - } - if (!device->pipeline_matmul_bf16) { - device->pipeline_matmul_bf16 = std::make_shared<vk_matmul_pipeline_struct>(); - } - if (!device->pipeline_matmul_id_bf16) { - device->pipeline_matmul_id_bf16 = std::make_shared<vk_matmul_pipeline_struct>(); - } + ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv6_f32, "rwkv_wkv6_f32", rwkv_wkv6_f32_len, rwkv_wkv6_f32_data, "main", 7, sizeof(vk_op_rwkv_wkv6_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); - auto const &ggml_vk_create_pipeline = [&](vk_device& device, vk_pipeline& base_pipeline, const char *name, size_t spv_size, const void* spv_data, const char *entrypoint, - uint32_t parameter_count, uint32_t push_constant_size, std::array<uint32_t, 3> wg_denoms, const std::vector<uint32_t>& specialization_constants, - uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) { + ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv7_f32, "rwkv_wkv7_f32", rwkv_wkv7_f32_len, rwkv_wkv7_f32_data, "main", 8, sizeof(vk_op_rwkv_wkv7_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); - if (!require_full_subgroups && required_subgroup_size == 0) { - required_subgroup_size = get_subgroup_size(name, device->architecture); - } + ggml_vk_create_pipeline(device, device->pipeline_gated_linear_attn_f32, "gated_linear_attn_f32", gated_linear_attn_f32_len, gated_linear_attn_f32_data, "main", 6, sizeof(vk_op_gated_linear_attn_push_constants), {1, 1, 1}, {}, 1); - vk_pipeline *ptr = &base_pipeline; + { + const bool li_subgroup = device->subgroup_arithmetic && device->subgroup_require_full_support; + const size_t li_len = li_subgroup ? lightning_indexer_subgroup_f32_len : lightning_indexer_f32_len; + const void * li_data = li_subgroup ? (const void *)lightning_indexer_subgroup_f32_data : (const void *)lightning_indexer_f32_data; - int num_pipelines = 1; -#if defined(VK_EXT_shader_64bit_indexing) - if (device->shader_64b_indexing) { - num_pipelines = 2; + for (ggml_type k_type : lightning_indexer_k_types) { + const std::string name = "lightning_indexer_" + std::string(ggml_type_name(k_type)) + "_k_f32"; + ggml_vk_create_pipeline(device, device->pipeline_lightning_indexer_f32[k_type], name.c_str(), li_len, li_data, "main", 5, sizeof(vk_op_lightning_indexer_push_constants), {1, 1, 1}, {(uint32_t)k_type, fa_block_bytes(k_type), device->subgroup_size}, 1, true, li_subgroup); } -#endif - for (int i = 0; i < num_pipelines; ++i, ptr = &(*ptr)->next) { - vk_pipeline &pipeline = *ptr; - if (!pipeline) { - pipeline = std::make_shared<vk_pipeline_struct>(); - } - if (!pipeline->initialized) { - pipeline->name = name; - pipeline->parameter_count = parameter_count; - pipeline->push_constant_size = push_constant_size; - pipeline->wg_denoms = wg_denoms; - pipeline->align = align; - pipeline->initialized = true; -#if defined(VK_EXT_shader_64bit_indexing) - pipeline->is_64b_indexing = (i == 1); -#endif - } + } - // We only care about the pipeline this call asked for; the rest - // (including the 64-bit indexing variant) are handled by their - // own request_descriptor_sets / load_shaders calls. - if (pipeline.get() != requested.get()) { - continue; + { + const uint32_t gdn_sizes[] = {16, 32, 64, 128}; + const char * gdn_names[][2] = { + {"gated_delta_net_f32_d16", "gated_delta_net_f32_d16_kda"}, + {"gated_delta_net_f32_d32", "gated_delta_net_f32_d32_kda"}, + {"gated_delta_net_f32_d64", "gated_delta_net_f32_d64_kda"}, + {"gated_delta_net_f32_d128", "gated_delta_net_f32_d128_kda"}, + }; + const char * gdn_chunked_names[] = { + "gated_delta_net_chunked_f32_d16", + "gated_delta_net_chunked_f32_d32", + "gated_delta_net_chunked_f32_d64", + "gated_delta_net_chunked_f32_d128", + }; + for (uint32_t si = 0; si < 4; si++) { + const uint32_t S_V = gdn_sizes[si]; + GGML_ASSERT(is_pow2(S_V)); + + uint32_t lanes_per_column; + if (S_V >= 128u && device->subgroup_clustered) { + lanes_per_column = 8u; + } else { + // Use largest power-of-two that divides both S_V and subgroup_size so that + // (1) S_V % lanes_per_column == 0 and (2) S_V % (subgroup_size / lanes_per_column) == 0. + // This means we don't need extra bounds checking logic in the shader. + lanes_per_column = std::min(S_V, device->subgroup_size); } - if (pipeline->compiled) { - continue; + // gated_delta_net.comp relies on S_V % COLS_PER_WG == 0 and + // S_V % LANES_PER_COLUMN == 0 to avoid bounds checks. + while (lanes_per_column > 1u) { + const bool valid_lanes = (device->subgroup_size % lanes_per_column) == 0 && + (S_V % lanes_per_column) == 0; + const uint32_t cols_per_wg = valid_lanes ? device->subgroup_size / lanes_per_column : 0; + if (valid_lanes && cols_per_wg > 0 && (S_V % cols_per_wg) == 0) { + break; + } + lanes_per_column >>= 1u; } - wait_pipeline = pipeline; + GGML_ASSERT((device->subgroup_size % lanes_per_column) == 0); + GGML_ASSERT((S_V % lanes_per_column) == 0); + GGML_ASSERT((S_V % (device->subgroup_size / lanes_per_column)) == 0); - if (!pipeline->compile_pending) { - pipeline->compile_pending = true; - claimed_task.pipeline = pipeline; - claimed_task.spv_size = spv_size; - claimed_task.spv_data = spv_data; - claimed_task.entrypoint = entrypoint; - claimed_task.parameter_count = parameter_count; - claimed_task.wg_denoms = wg_denoms; - claimed_task.specialization_constants = specialization_constants; - claimed_task.disable_robustness = disable_robustness; - claimed_task.require_full_subgroups = require_full_subgroups; - claimed_task.required_subgroup_size = required_subgroup_size; - has_claimed_task = true; + const bool need_partial_subgroup_reduce = lanes_per_column != 1u && lanes_per_column < device->subgroup_size; + const bool use_clustered_reduce = device->subgroup_arithmetic && device->subgroup_clustered && need_partial_subgroup_reduce; + const bool use_subgroup_reduce = device->subgroup_arithmetic && !need_partial_subgroup_reduce; + // Barrier-free fallback: the shared-memory reduction costs three workgroup + // barriers plus an LDS round trip per butterfly level, which dominates the + // per-token cost of the recurrence on older GPUs. Subgroup shuffle needs + // neither and is available on every GCN/RDNA driver. + const bool use_shuffle_reduce = !use_clustered_reduce && !use_subgroup_reduce && device->subgroup_shuffle; + const bool use_subgroup_ops = use_clustered_reduce || use_subgroup_reduce || use_shuffle_reduce; + size_t gdn_len; + const void * gdn_data; + if (use_clustered_reduce) { + gdn_len = gated_delta_net_f32_len; + gdn_data = (const void *)gated_delta_net_f32_data; + } else if (use_subgroup_reduce) { + gdn_len = gated_delta_net_f32_nocluster_len; + gdn_data = (const void *)gated_delta_net_f32_nocluster_data; + } else if (use_shuffle_reduce) { + gdn_len = gated_delta_net_f32_shuffle_len; + gdn_data = (const void *)gated_delta_net_f32_shuffle_data; + } else { + gdn_len = gated_delta_net_f32_shmem_len; + gdn_data = (const void *)gated_delta_net_f32_shmem_data; } - } - }; - auto const &ggml_vk_create_pipeline2 = [&](vk_device& device, vk_pipeline& pipeline, const std::string &name, size_t spv_size, const void* spv_data, const char *entrypoint, - uint32_t parameter_count, uint32_t push_constant_size, std::array<uint32_t, 3> wg_denoms, const std::vector<uint32_t>& specialization_constants, - uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) { - return ggml_vk_create_pipeline(device, pipeline, name.c_str(), spv_size, spv_data, entrypoint, - parameter_count, push_constant_size, wg_denoms, specialization_constants, - align, disable_robustness, require_full_subgroups, required_subgroup_size); - }; - - // FA scalar has two SPIR-V modules (MMQ vs non-MMQ); FA cm1 has one. K/V - // quant type is selected at runtime via the FaTypeK / FaTypeV spec constants. - - for (auto &fa : device->pipeline_flash_attn_f32_f16) { - if (fa.first.path != FA_SCALAR) continue; - const uint32_t Br = fa.first.Br; - const uint32_t Bc = fa.first.Bc; - const bool aligned = fa.first.aligned; - const bool f32acc = fa.first.f32acc; - const uint32_t fa_sgs = fa.first.subgroup_size; - const bool fa_ds = fa.first.subgroup_size == 0; + const uint32_t cols_per_wg = device->subgroup_size / lanes_per_column; + const std::array<uint32_t, 3> wg_denoms = {1u, 1u, cols_per_wg}; - const bool bf16_kv = fa.first.k_type == GGML_TYPE_BF16; - const bool use_mmq = ggml_vk_fa_scalar_uses_mmq(device, fa.first.k_type, fa.first.v_type); - const void * spv_data = nullptr; - size_t spv_size = 0; - const char *name = nullptr; - if (bf16_kv) { - spv_data = flash_attn_f32_f16_fp32_data; - spv_size = flash_attn_f32_f16_fp32_len; - name = aligned ? "flash_attn_f32_bf16_aligned" : "flash_attn_f32_bf16"; - } else if (use_mmq) { -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->fp16) { - if (f32acc) { spv_data = flash_attn_f32_f16_int8_data; spv_size = flash_attn_f32_f16_int8_len; } - else { spv_data = flash_attn_f32_f16_f16acc_int8_data; spv_size = flash_attn_f32_f16_f16acc_int8_len; } - } else { - spv_data = flash_attn_f32_f16_fp32_int8_data; - spv_size = flash_attn_f32_f16_fp32_int8_len; + for (uint32_t kda = 0; kda < 2; kda++) { + ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net[si][kda], + gdn_names[si][kda], gdn_len, gdn_data, "main", 7, sizeof(vk_op_gated_delta_net_push_constants), + wg_denoms, {S_V, kda, device->subgroup_size, lanes_per_column}, 1, true, use_subgroup_ops, device->subgroup_size); } -#endif - name = aligned ? "flash_attn_f32_f16_aligned" : "flash_attn_f32_f16"; - } else { - if (device->fp16) { - if (device->dot2_f16) { - if (f32acc) { spv_data = flash_attn_f32_f16_dot2_data; spv_size = flash_attn_f32_f16_dot2_len; } - else { spv_data = flash_attn_f32_f16_dot2_f16acc_data; spv_size = flash_attn_f32_f16_dot2_f16acc_len; } - } else { - if (f32acc) { spv_data = flash_attn_f32_f16_data; spv_size = flash_attn_f32_f16_len; } - else { spv_data = flash_attn_f32_f16_f16acc_data; spv_size = flash_attn_f32_f16_f16acc_len; } + + // Chunked (UT-transform) variant for short token blocks. Uses subgroup shuffle + // butterflies only, so it needs neither subgroup arithmetic nor clustered ops. + // The workgroup spans several subgroups to amortise the LDS staging of k/q and + // the Gram blocks over more output columns. + device->gated_delta_net_chunked_cols[si] = 0; + if (device->subgroup_shuffle && device->subgroup_require_full_support) { + uint32_t nsg = 1; + while (nsg < 4 && + cols_per_wg * nsg * 2 <= S_V && + (S_V % (cols_per_wg * nsg * 2)) == 0 && + device->subgroup_size * nsg * 2 <= device->properties.limits.maxComputeWorkGroupInvocations) { + nsg *= 2; } - } else { - spv_data = flash_attn_f32_f16_fp32_data; - spv_size = flash_attn_f32_f16_fp32_len; - } - name = aligned ? "flash_attn_f32_f16_aligned" : "flash_attn_f32_f16"; - } - ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 7, - sizeof(vk_flash_attn_push_constants), {Br, 1, 1}, - get_fa_spec_constants(fa.first), aligned ? Bc : 1, true, - !fa_ds, !fa_ds ? fa_sgs : 0); - } -#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - if (device->coopmat1_fa_support) { - for (auto &fa : device->pipeline_flash_attn_f32_f16) { - if (fa.first.path != FA_COOPMAT1) continue; - const uint32_t Br = fa.first.Br; - const uint32_t Bc = fa.first.Bc; - const bool aligned = fa.first.aligned; - const bool f32acc = fa.first.f32acc; - const uint32_t fa_sgs = fa.first.subgroup_size; - const bool fa_ds = fa.first.subgroup_size == 0; + const uint32_t cols_per_block = cols_per_wg * nsg; - const bool bf16_kv = fa.first.k_type == GGML_TYPE_BF16; + // 2 staging buffers of T_MAX*S_V floats + the two T_MAX^2 Gram blocks + 3*T_MAX scalars + const size_t chunked_lds = (size_t) 2 * 16 * S_V * sizeof(float) + + (size_t) 2 * 16 * 16 * sizeof(float) + + (size_t) 3 * 16 * sizeof(float); - const void * spv_data; - size_t spv_size; - const char *name; - if (bf16_kv) { -#if defined(VK_KHR_shader_bfloat16) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - if (!device->coopmat_bf16_support) continue; - spv_data = flash_attn_f32_f16_bf16_cm1_data; - spv_size = flash_attn_f32_f16_bf16_cm1_len; - name = aligned ? "flash_attn_f32_bf16_aligned_cm1" : "flash_attn_f32_bf16_cm1"; -#else - continue; -#endif - } else { - if (f32acc) { spv_data = flash_attn_f32_f16_cm1_data; spv_size = flash_attn_f32_f16_cm1_len; } - else { spv_data = flash_attn_f32_f16_f16acc_cm1_data; spv_size = flash_attn_f32_f16_f16acc_cm1_len; } - name = aligned ? "flash_attn_f32_f16_aligned_cm1" : "flash_attn_f32_f16_cm1"; + if ((S_V % cols_per_block) == 0 && chunked_lds <= device->properties.limits.maxComputeSharedMemorySize) { + const std::array<uint32_t, 3> chunked_wg_denoms = {1u, 1u, cols_per_block}; + ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_chunked[si], + gdn_chunked_names[si], gated_delta_net_chunked_f32_len, gated_delta_net_chunked_f32_data, + "main", 7, sizeof(vk_op_gated_delta_net_push_constants), + chunked_wg_denoms, {S_V, device->subgroup_size, lanes_per_column, nsg}, + 1, true, true, device->subgroup_size); + device->gated_delta_net_chunked_cols[si] = cols_per_block; + } } - ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 7, - sizeof(vk_flash_attn_push_constants), {Br, 1, 1}, - get_fa_spec_constants(fa.first), aligned ? Bc : 1, true, - !fa_ds, !fa_ds ? fa_sgs : 0); } } -#endif - -#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) - if (device->coopmat2) { - for (auto &fa : device->pipeline_flash_attn_f32_f16) { - if (fa.first.path != FA_COOPMAT2) continue; - const uint32_t Br = fa.first.Br; - const uint32_t Bc = fa.first.Bc; - const bool aligned = fa.first.aligned; - const bool f32acc = fa.first.f32acc; - const bool bf16_kv = fa.first.k_type == GGML_TYPE_BF16; - const void * spv_data; - size_t spv_size; - const char * name; - if (bf16_kv) { -#if defined(VK_KHR_shader_bfloat16) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - if (!device->coopmat2_bf16_support) continue; - spv_data = flash_attn_f32_f16_bf16_cm2_data; - spv_size = flash_attn_f32_f16_bf16_cm2_len; - name = aligned ? "flash_attn_f32_bf16_aligned_cm2" : "flash_attn_f32_bf16_cm2"; -#else - continue; -#endif - } else if (aligned) { - if (f32acc) { spv_data = flash_attn_f32_f16_cm2_data; spv_size = flash_attn_f32_f16_cm2_len; name = "flash_attn_f32_f16_aligned_f32acc_cm2"; } - else { spv_data = flash_attn_f32_f16_f16acc_cm2_data; spv_size = flash_attn_f32_f16_f16acc_cm2_len; name = "flash_attn_f32_f16_aligned_f16acc_cm2"; } - } else { - if (f32acc) { spv_data = flash_attn_f32_f16_cm2_data; spv_size = flash_attn_f32_f16_cm2_len; name = "flash_attn_f32_f16_f32acc_cm2"; } - else { spv_data = flash_attn_f32_f16_f16acc_cm2_data; spv_size = flash_attn_f32_f16_f16acc_cm2_len; name = "flash_attn_f32_f16_f16acc_cm2"; } - } - ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 7, - sizeof(vk_flash_attn_push_constants), {Br, 1, 1}, - get_fa_spec_constants(fa.first), aligned ? Bc : 1, true, false, 0); - } + if (device->subgroup_arithmetic && device->subgroup_require_full_support) { + ggml_vk_create_pipeline(device, device->pipeline_ssm_scan_f32_d128, "ssm_scan_128_f32", ssm_scan_subgroup_f32_len, ssm_scan_subgroup_f32_data, "main", 8, sizeof(vk_op_ssm_scan_push_constants), {1, 1, 1}, {128, device->subgroup_size}, 1, true, true); + ggml_vk_create_pipeline(device, device->pipeline_ssm_scan_f32_d256, "ssm_scan_256_f32", ssm_scan_subgroup_f32_len, ssm_scan_subgroup_f32_data, "main", 8, sizeof(vk_op_ssm_scan_push_constants), {1, 1, 1}, {256, device->subgroup_size}, 1, true, true); + } else { + ggml_vk_create_pipeline(device, device->pipeline_ssm_scan_f32_d128, "ssm_scan_128_f32", ssm_scan_f32_len, ssm_scan_f32_data, "main", 8, sizeof(vk_op_ssm_scan_push_constants), {1, 1, 1}, {128, device->subgroup_size, 16}, 1, true, true); + ggml_vk_create_pipeline(device, device->pipeline_ssm_scan_f32_d256, "ssm_scan_256_f32", ssm_scan_f32_len, ssm_scan_f32_data, "main", 8, sizeof(vk_op_ssm_scan_push_constants), {1, 1, 1}, {256, device->subgroup_size, 16}, 1, true, true); } -#endif - auto const &ggml_vk_mul_mm_spec = [&device](std::vector<uint32_t> spec, bool aligned) { - spec.push_back(aligned ? 1u : 0u); // constantID=11: ALIGNED - if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && - device->driver_id == vk::DriverId::eIntelProprietaryWindows) { - spec.push_back(0u); // constantID=12: SHMEM_STRIDE_PAD = 0 - spec.push_back(1u); // constantID=13: APPLY_SLM_A_RESHAPE = true - } - return spec; - }; + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_f32, "ssm_conv_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 4, sizeof(vk_op_ssm_conv_push_constants), {32, 16, 1}, {32, 16, 0, 0}, 1); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_silu_f32, "ssm_conv_silu_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 4, sizeof(vk_op_ssm_conv_push_constants), {32, 16, 1}, {32, 16, 0, 1}, 1); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_bias_silu_f32, "ssm_conv_bias_silu_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 4, sizeof(vk_op_ssm_conv_push_constants), {32, 16, 1}, {32, 16, 1, 1}, 1); - // 5 stock + 1 for the weight-paging per-expert offset array (binding 5). - const int mul_mat_id_param_count = 6; + ggml_vk_create_pipeline(device, device->pipeline_opt_step_adamw_f32, "opt_step_adamw_f32", opt_step_adamw_f32_len, opt_step_adamw_f32_data, "main", 5, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); -#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) - if (device->coopmat2) { - auto const &ggml_vk_mul_mm_cm2_spec = [](std::vector<uint32_t> spec, bool aligned, bool mul_mat_id) { - if (mul_mat_id && spec.size() > 5) { - spec.insert(spec.begin() + 5, aligned ? 1u : 0u); - } else { - spec.push_back(aligned ? 1u : 0u); - } - if (mul_mat_id && spec.size() == 6) { - spec.push_back(32); - } - return spec; - }; + ggml_vk_create_pipeline(device, device->pipeline_opt_step_sgd_f32, "opt_step_sgd_f32", opt_step_sgd_f32_len, opt_step_sgd_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - // Create 6 variants, {s,m,l}x{unaligned,aligned} -#define CREATE_MM(PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(l_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(m_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(s_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(l_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), l_align, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(m_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), m_align, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(s_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), s_align, true); \ - - // Create 2 variants, {f16,f32} accumulator -#define CREATE_MM2(PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - CREATE_MM(PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - CREATE_MM(PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - - CREATE_MM2(pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3) -#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - if (device->coopmat_bf16_support) { - CREATE_MM(pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3) + // conv2d, conv_transpose_2d, conv3d + for (uint32_t s = 0; s < CONV_SHAPE_COUNT; ++s) { + // smaller WG for the small-tile fallback gives more concurrent WGs per SM + uint32_t conv2d_WG_SIZE = (s == CONV_SHAPE_64x32) ? 128 : 256; + uint32_t use_collectives = 0; // Enables subgroup ops for preventing the re-calculation of indices. + uint32_t conv2d_TS_K = (s == CONV_SHAPE_64x32) ? 4 : 8; + uint32_t conv2d_SHMEM_PAD = 4; + vk_conv_block_size conv2d_BS = vk_conv_block_sizes[s]; + bool conv2d_UNROLL = true; + +#if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) + if (device->coopmat2) { + conv2d_SHMEM_PAD = 8; // 8 float16_t } #endif - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q1_0], matmul_q1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_0], matmul_q2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_0], matmul_q4_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_1], matmul_q4_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_0], matmul_q5_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_1], matmul_q5_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q8_0], matmul_q8_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_K], matmul_q2_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_TQ2_0], matmul_tq2_0_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q3_K], matmul_q3_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_K], matmul_q4_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_K], matmul_q5_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q6_K], matmul_q6_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_S], matmul_iq1_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_M], matmul_iq1_m_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_S], matmul_iq2_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_S], matmul_iq3_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4) { - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - } else -#endif - { - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) + + if (device->vendor_id == VK_VENDOR_ID_INTEL) { + conv2d_SHMEM_PAD = 0; + conv2d_UNROLL = false; + } else if (device->vendor_id == VK_VENDOR_ID_AMD) { + conv2d_SHMEM_PAD = device->architecture == vk_device_architecture::AMD_GCN ? 1 : 4; + if (s == CONV_SHAPE_128x128 && device->architecture != vk_device_architecture::AMD_GCN) { + conv2d_UNROLL = false; + } } - GGML_ASSERT(device->subgroup_ballot); + // Use collectives on pre-Turing NVIDIA GPUs and GCN AMD cards, which had slower integer math. + bool allow_collectives_nv = device->vendor_id != VK_VENDOR_ID_NVIDIA || + device->architecture == vk_device_architecture::NVIDIA_PRE_TURING; + bool allow_collectives_amd = device->vendor_id != VK_VENDOR_ID_AMD || + device->architecture == vk_device_architecture::AMD_GCN; - CREATE_MM2(pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, 5) -#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - if (device->coopmat_bf16_support) { - CREATE_MM(pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, 5) - } -#endif - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4) { - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16_ocp, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16_ocp, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - } else -#endif - { - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) + if (device->subgroup_shuffle && + device->vendor_id != VK_VENDOR_ID_INTEL && // Do not enable collectives on Intel, see PR 14316. + allow_collectives_nv && + allow_collectives_amd) { + use_collectives = 1; + conv2d_BS.CRS = std::min( + device->subgroup_size, + conv2d_BS.CRS); // CRS block size should be capped at subgroup size for correctness when shuffle is used. } -#undef CREATE_MM -#undef CREATE_MM2 - } else -#endif // defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) -#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - if (device->coopmat_support) { - // Create 6 variants, {s,m,l}x{unaligned,aligned} -#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, true); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, true); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, true); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, true); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, true); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, true); \ - - // Create 2 variants, {f16,f32} accumulator -#define CREATE_MM2(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - if (device->coopmat_acc_f16_support) { \ - CREATE_MM(TYPE, PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - } \ - if (device->coopmat_acc_f32_support) { \ - CREATE_MM(TYPE, PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - } \ - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16_f32, matmul_f16_f32, wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); -#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - if (device->coopmat_bf16_support) { - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, ) - } + // cm1 is used only when cm2 is unavailable; capped at 64x128 (due to shared memory size). + // Requires 16x16x16 f16-acc since that's the fragment shape hard-coded in the shader. + // Subgroup size must be 32 or 64 (to keep WG_SIZE sane) and we need + // subgroup_size_control to force the driver to actually use it. + bool conv2d_use_cm1 = false; +#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + conv2d_use_cm1 = !device->coopmat2 && + device->coopmat_support && device->coopmat_support_16x16x16_f16acc && + device->subgroup_size_control && + (device->subgroup_size == 32 || device->subgroup_size == 64) && + s != CONV_SHAPE_128x128; #endif - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0], matmul_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0], matmul_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0], matmul_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1], matmul_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0], matmul_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1], matmul_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0], matmul_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K], matmul_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0], matmul_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K], matmul_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K], matmul_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K], matmul_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K], matmul_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S], matmul_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M], matmul_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S], matmul_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S], matmul_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); + const uint32_t conv2d_cm1_shmem_pad = 8; -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4) { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - } else -#endif - { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - } + auto shmem_req = [&](uint32_t pad, bool csh_store, bool fp16_shmem) { + const uint32_t elem_size = fp16_shmem ? (uint32_t)sizeof(uint16_t) : (uint32_t)sizeof(float); + const uint32_t csh_elems = csh_store ? conv2d_BS.K * conv2d_BS.NPQ : 0u; + return (conv2d_BS.K * (conv2d_BS.CRS + pad) + conv2d_BS.CRS * (conv2d_BS.NPQ + pad) + csh_elems) * elem_size; + }; - GGML_ASSERT(device->subgroup_ballot); + // 2D, transpose-2D, and 3D conv use the same KxCRS @ CRSxNPQ shmem + // layout. cm1 needs Csh for output, so check before applying cm1 params. + if (conv2d_use_cm1 && device->properties.limits.maxComputeSharedMemorySize < shmem_req(conv2d_cm1_shmem_pad, true, true)) { + conv2d_use_cm1 = false; + } - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_subgroup_f16_f32, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); -#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - if (device->coopmat_bf16_support) { - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); + uint32_t conv2d_WM = 16, conv2d_WN = 16; // cm1 subgroup tile, ignored otherwise + if (conv2d_use_cm1) { + conv2d_SHMEM_PAD = conv2d_cm1_shmem_pad; + // 16x16x16 fragments; pick WM/WN to keep WG_SIZE at 256 + // (i.e. 8 subgroups for sg=32, 4 subgroups for sg=64). + const bool sg64 = (device->subgroup_size == 64); + switch (s) { + case CONV_SHAPE_64x32: conv2d_WM = sg64 ? 32 : 16; conv2d_WN = 16; break; + case CONV_SHAPE_64x128: conv2d_WM = 32; conv2d_WN = sg64 ? 64 : 32; break; + case CONV_SHAPE_32x256: conv2d_WM = sg64 ? 16 : 32; conv2d_WN = sg64 ? 128 : 32; break; + default: break; + } + const uint32_t warps_M = conv2d_BS.K / conv2d_WM; + const uint32_t warps_N = conv2d_BS.NPQ / conv2d_WN; + conv2d_WG_SIZE = warps_M * warps_N * device->subgroup_size; } -#endif - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4) { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - } else -#endif - { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); + // stage cm2 accumulator through shmem for coalesced global stores; + // skipped on 128x128 where the extra Csh footprint hurts occupancy. + // cm1 always uses the staged path. + uint32_t conv2d_csh_store = (device->coopmat2 && s != CONV_SHAPE_128x128) ? 1u : 0u; + if (conv2d_use_cm1) { + conv2d_csh_store = 1; } -#undef CREATE_MM2 -#undef CREATE_MM - } else -#endif // defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - if (device->fp16) { - // Create 6 variants, {s,m,l}x{unaligned,aligned} - // Selects dot2 SPIR-V variant at runtime when device->dot2_f16 is true -#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - - // bf16 scalar path promotes to f32, no dot2 variant -#define CREATE_MM_NODOT2(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - -#define CREATE_MMQ(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->l, #NAMELC "_l", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, l_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ - if (device->mul_mat ## ID ## _m_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->m, #NAMELC "_m", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, m_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ - if (device->mul_mat ## ID ## _s_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->s, #NAMELC "_s", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, s_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ - // Create 2 variants, {f16,f32} accumulator -#define CREATE_MM2(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - CREATE_MM(TYPE, PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - CREATE_MM(TYPE, PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16_f32, matmul_f16_f32, wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0], matmul_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0], matmul_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0], matmul_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1], matmul_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0], matmul_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1], matmul_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0], matmul_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K], matmul_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0], matmul_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K], matmul_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K], matmul_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K], matmul_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K], matmul_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S], matmul_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M], matmul_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S], matmul_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S], matmul_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); + // shmem is fp16 on cm2/cm1 (matches Csh), fp32 on scalar + const bool conv2d_use_fp16_shmem = device->coopmat2 || conv2d_use_cm1; -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_0], matmul_q2_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_0], matmul_q4_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_1], matmul_q4_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_0], matmul_q5_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_1], matmul_q5_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q8_0], matmul_q8_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); + // shrink CRS if the non-cm1 config still doesn't fit + if (device->properties.limits.maxComputeSharedMemorySize < shmem_req(conv2d_SHMEM_PAD, conv2d_csh_store, conv2d_use_fp16_shmem)) { + GGML_ASSERT(!conv2d_use_cm1); + conv2d_BS.CRS = 8; + if (use_collectives) { + conv2d_BS.CRS = std::min(device->subgroup_size, conv2d_BS.CRS); + } + conv2d_csh_store = 0; + } + + std::array<uint32_t, 3> wg_denoms = { conv2d_BS.K, 1, 1 }; + std::vector<uint32_t> spec_constants = { conv2d_WG_SIZE, conv2d_BS.K, conv2d_BS.CRS, conv2d_BS.NPQ, conv2d_TS_K, use_collectives, conv2d_SHMEM_PAD }; - CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_MXFP4], matmul_mxfp4_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); + // cm1 needs a fixed subgroup width to match the WG_SIZE we computed + const uint32_t conv2d_required_subgroup_size = conv2d_use_cm1 ? device->subgroup_size : 0; - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_K], matmul_q2_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q3_K], matmul_q3_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_K], matmul_q4_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_K], matmul_q5_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q6_K], matmul_q6_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); +#define CREATE_CONV(name, type_suffix, spv_suffix) \ + for (auto &c : device->pipeline_##name##type_suffix[s]) { \ + const vk_conv2d_pipeline_state &state = c.first; \ + std::vector<uint32_t> spec_constants_cpy = spec_constants; \ + spec_constants_cpy.push_back(state.s0); \ + spec_constants_cpy.push_back(state.s1); \ + spec_constants_cpy.push_back(state.p0); \ + spec_constants_cpy.push_back(state.p1); \ + spec_constants_cpy.push_back(state.d0); \ + spec_constants_cpy.push_back(state.d1); \ + spec_constants_cpy.push_back(state.KW); \ + spec_constants_cpy.push_back(state.KH); \ + spec_constants_cpy.push_back(state.aligned); \ + spec_constants_cpy.push_back(conv2d_csh_store); \ + spec_constants_cpy.push_back(conv2d_WM); \ + spec_constants_cpy.push_back(conv2d_WN); \ + ggml_vk_create_pipeline( \ + device, c.second, #name #type_suffix, \ + name##type_suffix##spv_suffix##_len, name##type_suffix##spv_suffix##_data, "main", 3, \ + sizeof(vk_op_conv2d_push_constants), wg_denoms, spec_constants_cpy, 1, true, use_collectives || conv2d_required_subgroup_size, conv2d_required_subgroup_size); \ } +#define CREATE_CONVS(spv_suffix) \ + CREATE_CONV(conv2d, _f32, spv_suffix) \ + CREATE_CONV(conv2d, _f16_f32, spv_suffix) \ + CREATE_CONV(conv_transpose_2d, _f32, spv_suffix) \ + CREATE_CONV(conv_transpose_2d, _f16_f32, spv_suffix) +#if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) + if (device->coopmat2) { + CREATE_CONVS(_cm2) + } else #endif - - if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_subgroup_f16_f32, wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - - CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - } +#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + if (conv2d_use_cm1) { + CREATE_CONVS(_cm1) + } else #endif + if (conv2d_UNROLL) { + CREATE_CONVS(_unroll) } else { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_f16_f32, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_q2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_q5_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_q5_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_q8_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_q2_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_tq2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_q3_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_q4_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_q5_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_q6_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_0], matmul_id_q2_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_0], matmul_id_q4_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_1], matmul_id_q4_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_0], matmul_id_q5_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_1], matmul_id_q5_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q8_0], matmul_id_q8_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_MXFP4], matmul_id_mxfp4_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_K], matmul_id_q2_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q3_K], matmul_id_q3_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_K], matmul_id_q4_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_K], matmul_id_q5_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q6_K], matmul_id_q6_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - } -#endif + CREATE_CONVS( ) } -#undef CREATE_MM2 -#undef CREATE_MMQ -#undef CREATE_MM -#undef CREATE_MM_NODOT2 - } else { - // Create 6 variants, {s,m,l}x{unaligned,aligned} -#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - -#define CREATE_MMQ(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - if (device->mul_mat ## ID ## _l_int[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC "_l", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, l_ ## WARPTILE, 1); \ - if (device->mul_mat ## ID ## _m_int[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC "_m", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, m_ ## WARPTILE, 1); \ - if (device->mul_mat ## ID ## _s_int[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC "_s", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, s_ ## WARPTILE, 1); \ - - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_f16.f32acc, matmul_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_f16_f32.f32acc, matmul_f16_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0].f32acc, matmul_q1_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0].f32acc, matmul_q2_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0].f32acc, matmul_q4_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1].f32acc, matmul_q4_1_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0].f32acc, matmul_q5_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1].f32acc, matmul_q5_1_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0].f32acc, matmul_q8_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K].f32acc, matmul_q2_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0].f32acc, matmul_tq2_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K].f32acc, matmul_q3_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K].f32acc, matmul_q4_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K].f32acc, matmul_q5_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K].f32acc, matmul_q6_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S].f32acc, matmul_iq1_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M].f32acc, matmul_iq1_m_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS].f32acc, matmul_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS].f32acc, matmul_iq2_xs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S].f32acc, matmul_iq2_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS].f32acc, matmul_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S].f32acc, matmul_iq3_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS].f32acc, matmul_iq4_xs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL].f32acc, matmul_iq4_nl_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4].f32acc, matmul_mxfp4_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4].f32acc, matmul_nvfp4_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); +#undef CREATE_CONV +#undef CREATE_CONVS -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_0].f32acc, matmul_q2_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_0].f32acc, matmul_q4_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_1].f32acc, matmul_q4_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_0].f32acc, matmul_q5_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_1].f32acc, matmul_q5_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q8_0].f32acc, matmul_q8_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_K].f32acc, matmul_q2_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q3_K].f32acc, matmul_q3_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_K].f32acc, matmul_q4_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_K].f32acc, matmul_q5_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q6_K].f32acc, matmul_q6_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); + std::vector<uint32_t> conv3d_spec_constants = { conv2d_WG_SIZE, conv2d_BS.K, conv2d_BS.CRS, conv2d_BS.NPQ, conv2d_TS_K, conv2d_SHMEM_PAD }; +#define CREATE_CONV3D(type_suffix, spv_suffix) \ + for (auto &c : device->pipeline_conv3d##type_suffix[s]) { \ + const vk_conv3d_pipeline_state &state = c.first; \ + std::vector<uint32_t> spec_constants_cpy = conv3d_spec_constants; \ + spec_constants_cpy.push_back(state.s0); \ + spec_constants_cpy.push_back(state.s1); \ + spec_constants_cpy.push_back(state.s2); \ + spec_constants_cpy.push_back(state.p0); \ + spec_constants_cpy.push_back(state.p1); \ + spec_constants_cpy.push_back(state.p2); \ + spec_constants_cpy.push_back(state.d0); \ + spec_constants_cpy.push_back(state.d1); \ + spec_constants_cpy.push_back(state.d2); \ + spec_constants_cpy.push_back(state.KW); \ + spec_constants_cpy.push_back(state.KH); \ + spec_constants_cpy.push_back(state.KD); \ + spec_constants_cpy.push_back(state.aligned); \ + spec_constants_cpy.push_back(conv2d_csh_store); \ + spec_constants_cpy.push_back(conv2d_WM); \ + spec_constants_cpy.push_back(conv2d_WN); \ + ggml_vk_create_pipeline( \ + device, c.second, "conv3d" #type_suffix, \ + conv3d##type_suffix##spv_suffix##_len, conv3d##type_suffix##spv_suffix##_data, "main", 3, \ + sizeof(vk_op_conv3d_push_constants), wg_denoms, spec_constants_cpy, 1, true, conv2d_required_subgroup_size != 0, conv2d_required_subgroup_size); \ } +#if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) + if (device->coopmat2) { + CREATE_CONV3D(_f32, _cm2) + CREATE_CONV3D(_f16_f32, _cm2) + } else #endif - - if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16.f32acc, matmul_id_subgroup_f16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16_f32.f32acc, matmul_id_subgroup_f16_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - - CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_subgroup_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0].f32acc, matmul_id_subgroup_q2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_subgroup_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_subgroup_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0].f32acc, matmul_id_subgroup_q5_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1].f32acc, matmul_id_subgroup_q5_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0].f32acc, matmul_id_subgroup_q8_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K].f32acc, matmul_id_subgroup_q2_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0].f32acc, matmul_id_subgroup_tq2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K].f32acc, matmul_id_subgroup_q3_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K].f32acc, matmul_id_subgroup_q4_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K].f32acc, matmul_id_subgroup_q5_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K].f32acc, matmul_id_subgroup_q6_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S].f32acc, matmul_id_subgroup_iq1_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M].f32acc, matmul_id_subgroup_iq1_m_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS].f32acc, matmul_id_subgroup_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS].f32acc, matmul_id_subgroup_iq2_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S].f32acc, matmul_id_subgroup_iq2_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS].f32acc, matmul_id_subgroup_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S].f32acc, matmul_id_subgroup_iq3_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS].f32acc, matmul_id_subgroup_iq4_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL].f32acc, matmul_id_subgroup_iq4_nl_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4].f32acc, matmul_id_subgroup_mxfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4].f32acc, matmul_id_subgroup_nvfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - } else { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16.f32acc, matmul_id_f16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16_f32.f32acc, matmul_id_f16_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0].f32acc, matmul_id_q2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0].f32acc, matmul_id_q5_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1].f32acc, matmul_id_q5_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0].f32acc, matmul_id_q8_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K].f32acc, matmul_id_q2_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0].f32acc, matmul_id_tq2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K].f32acc, matmul_id_q3_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K].f32acc, matmul_id_q4_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K].f32acc, matmul_id_q5_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K].f32acc, matmul_id_q6_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S].f32acc, matmul_id_iq1_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M].f32acc, matmul_id_iq1_m_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS].f32acc, matmul_id_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS].f32acc, matmul_id_iq2_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S].f32acc, matmul_id_iq2_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS].f32acc, matmul_id_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S].f32acc, matmul_id_iq3_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS].f32acc, matmul_id_iq4_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL].f32acc, matmul_id_iq4_nl_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4].f32acc, matmul_id_mxfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4].f32acc, matmul_id_nvfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - } - } - // reusing CREATE_MM from the fp32 path - if ((device->coopmat2 || device->coopmat_support) -#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - && !device->coopmat_bf16_support +#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + if (conv2d_use_cm1) { + CREATE_CONV3D(_f32, _cm1) + CREATE_CONV3D(_f16_f32, _cm1) + } else #endif - ) { - const uint32_t s_warptile_wm = device->subgroup_size == 8 ? 8 : 32; - - // use scalar tile sizes - l_warptile = { 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 }; - m_warptile = { 128, 64, 64, 16, mm_warp_8, 32, 2, 4, 2, 1, mm_warp_8 }; - s_warptile = { subgroup_size_32, 32, 32, 16, s_warptile_wm, 32, 2, 2, 2, 1, subgroup_size_8 }; - - l_wg_denoms = {128, 128, 1 }; - m_wg_denoms = { 64, 64, 1 }; - s_wg_denoms = { 32, 32, 1 }; - - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); + if (conv2d_UNROLL) { + CREATE_CONV3D(_f32, _unroll) + CREATE_CONV3D(_f16_f32, _unroll) + } else { + CREATE_CONV3D(_f32, ) + CREATE_CONV3D(_f16_f32, ) + } +#undef CREATE_CONV3D } -#undef CREATE_MM - // mul mat vec + ggml_vk_create_pipeline(device, device->pipeline_conv2d_dw_whcn_f32, "conv2d_dw_whcn_f32", conv2d_dw_whcn_f32_len, conv2d_dw_whcn_f32_data, "main", 3, sizeof(vk_op_conv2d_dw_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_conv2d_dw_cwhn_f32, "conv2d_dw_cwhn_f32", conv2d_dw_cwhn_f32_len, conv2d_dw_cwhn_f32_data, "main", 3, sizeof(vk_op_conv2d_dw_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_conv2d_dw_whcn_f16_f32, "conv2d_dw_whcn_f16_f32", conv2d_dw_whcn_f16_f32_len, conv2d_dw_whcn_f16_f32_data, "main", 3, sizeof(vk_op_conv2d_dw_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_conv2d_dw_cwhn_f16_f32, "conv2d_dw_cwhn_f16_f32", conv2d_dw_cwhn_f16_f32_len, conv2d_dw_cwhn_f16_f32_data, "main", 3, sizeof(vk_op_conv2d_dw_push_constants), {512, 1, 1}, {}, 1); - // the number of rows computed per shader depends on GPU model and quant - uint32_t rm_stdq = 1; - uint32_t rm_kq = 2; - uint32_t rm_stdq_int = 1; - uint32_t rm_kq_int = 1; - // Rows per workgroup for the BF16 mat-vec shader. It takes the - // iter_aligned_nonquant path, where after the rows-outside restructure each - // A load is fanned across all NUM_COLS columns. More rows per workgroup then - // amortizes the per-column B loads and the final reduction over more useful - // work, which is what the multi-column BF16 GEMV needs on GCN. Left at the - // stock 2 elsewhere so RDNA/NVIDIA/Intel keep byte-identical pipelines. - uint32_t rm_bf16 = 2; - auto const &rm_iq_int = [](uint32_t i) { return i == 0 ? 8u : 4u; }; - if (device->vendor_id == VK_VENDOR_ID_AMD) { - if (device->architecture == AMD_GCN) { - rm_stdq = 2; - rm_kq = 4; - rm_stdq_int = 4; - rm_bf16 = 4; + for (uint32_t use_push = 0; use_push < 2; ++use_push) { + for (uint32_t i = 0; i < num_topk_moe_pipelines; ++i) { + ggml_vk_create_pipeline2(device, device->pipeline_topk_moe[i][use_push], "topk_moe_f32_"+std::to_string(i), topk_moe_f32_len, topk_moe_f32_data, "main", 4, sizeof(vk_op_topk_moe_push_constants), {1, 1, 1}, {device->subgroup_size, 1u<<i, use_push}, 1, true, true, device->subgroup_size); } - } else if (device->vendor_id == VK_VENDOR_ID_INTEL) { - rm_stdq = 2; - rm_stdq_int = 2; } - // RDNA3: above four columns, static 4 rows for all types bench faster than the default - const bool is_rdna3 = device->vendor_id == VK_VENDOR_ID_AMD && device->architecture == AMD_RDNA3; - auto const &rm_int_n = [&](uint32_t rows, uint32_t i) { return (is_rdna3 && i >= 4) ? 4u : rows; }; - // RDNA3: Static 4 rows for all types bench faster than the default - auto const &rm_id = [&](uint32_t rows) { return is_rdna3 ? 4u : rows; }; - uint32_t rm_iq = 2 * rm_kq; - // How wide a batch still belongs on the mat-vec path rather than mul_mm. - // GCN goes to 16: with the A load hoisted out of the column loop the GEMV - // cost is close to flat in NUM_COLS, so it stays well under the mul_mm - // crossover that otherwise bites at n=9. Every other architecture keeps the - // stock 8 -- both because the crossover has not been measured there and - // because each extra column instantiates another pipeline per type. - device->mul_mat_vec_max_cols_eff = mul_mat_vec_max_cols_default; - if (device->vendor_id == VK_VENDOR_ID_AMD && device->architecture == AMD_GCN) { - device->mul_mat_vec_max_cols_eff = 16; - } - // Escape hatch for A/B measurement without a rebuild. - if (const char * e = getenv("GGML_VK_MMV_MAX_COLS")) { - const int v = atoi(e); - if (v >= 1 && v <= (int)mul_mat_vec_max_cols) { - device->mul_mat_vec_max_cols_eff = (uint32_t)v; - } - } - // Escape hatch: rows per workgroup for K-quants / standard quants. - // rm changes workgroup ownership, not a row's k-reduction order. - if (const char * e = getenv("GGML_VK_MMV_ROWS_KQ")) { - const int v = atoi(e); - if (v >= 1 && v <= 64) { - rm_kq = (uint32_t)v; - rm_iq = 2 * rm_kq; - } - } - if (const char * e = getenv("GGML_VK_MMV_ROWS_STDQ")) { - const int v = atoi(e); - if (v >= 1 && v <= 64) { - rm_stdq = (uint32_t)v; - } - } - if (requested == nullptr) { - GGML_LOG_INFO("ggml_vulkan: %s mmv rows rm_kq=%u rm_stdq=%u\n", - device->name.c_str(), rm_kq, rm_stdq); + ggml_vk_create_pipeline(device, device->pipeline_wp_fused_expert_q5_1, + "wp_fused_expert_q5_1", wp_fused_expert_q5_1_len, + wp_fused_expert_q5_1_data, "main", 6, + sizeof(vk_wp_fused_expert_push_constants), {1, 1, 1}, {}, 1, + true, true, 64); + ggml_vk_create_pipeline(device, device->pipeline_wp_fused_expert_q8_0, + "wp_fused_expert_q8_0", wp_fused_expert_q8_0_len, + wp_fused_expert_q8_0_data, "main", 6, + sizeof(vk_wp_fused_expert_push_constants), {1, 1, 1}, {}, 1, + true, true, 64); + // Phase A: 5 bindings -- gate, up, input, params, scratch (no down + // buffer, no output buffer; it only ever writes the scratch buffer). + ggml_vk_create_pipeline(device, device->pipeline_wp_fused_expert_batch_a, + "wp_fused_expert_batch_a", wp_fused_expert_batch_a_len, + wp_fused_expert_batch_a_data, "main", 5, + sizeof(vk_wp_fused_expert_batch_push_constants), {1, 1, 1}, {}, 1, + true, true, 64); + // Phase B: 4 bindings -- down, output, params, scratch (no gate/up/input + // buffer; it only ever reads the scratch buffer phase A wrote). + ggml_vk_create_pipeline(device, device->pipeline_wp_fused_expert_batch_b_q5_1, + "wp_fused_expert_batch_b_q5_1", wp_fused_expert_batch_b_q5_1_len, + wp_fused_expert_batch_b_q5_1_data, "main", 4, + sizeof(vk_wp_fused_expert_batch_push_constants), {1, 1, 1}, {}, 1, + true, true, 64); + ggml_vk_create_pipeline(device, device->pipeline_wp_fused_expert_batch_b_q8_0, + "wp_fused_expert_batch_b_q8_0", wp_fused_expert_batch_b_q8_0_len, + wp_fused_expert_batch_b_q8_0_data, "main", 4, + sizeof(vk_wp_fused_expert_batch_push_constants), {1, 1, 1}, {}, 1, + true, true, 64); + + // Drop compile_mutex so other threads can walk while we compile. + compile_lock.unlock(); + + // Compile what we claimed; create_pipeline_func reacquires compile_mutex + // at the end to flip compile_pending/compiled and notify waiters. + if (has_claimed_task) { + auto & task = claimed_task; + ggml_vk_create_pipeline_func(device, task.pipeline, task.spv_size, task.spv_data, + task.entrypoint, task.parameter_count, task.wg_denoms, + task.specialization_constants, task.disable_robustness, + task.require_full_subgroups, task.required_subgroup_size); } - const bool use_subgroups = device->subgroup_arithmetic; - // Ensure a subgroup size >= 16 is available - const bool use_subgroups16 = use_subgroups && subgroup_min_size_16; + // Another thread may be compiling the pipeline we need; block on it here. + if (wait_pipeline) { + std::unique_lock<std::mutex> wait_lock(device->compile_mutex); + device->compile_cv.wait(wait_lock, [&] { + return wait_pipeline->compiled.load(); + }); + } +} - const uint32_t subgroup_size = (device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control && device->subgroup_min_size <= 16 && device->subgroup_max_size >= 16) ? 16 : device->subgroup_size; - const uint32_t subgroup_size16 = std::max(subgroup_size, 16u); +vk_device ggml_vk_get_device(size_t idx) { + VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")"); - const uint32_t force_subgroup_size = use_subgroups ? subgroup_size : 0; - const uint32_t force_subgroup_size16 = use_subgroups16 ? subgroup_size16 : 0; - static constexpr uint32_t mul_mat_vec_num_bindings = 5; - // 6 stock + 1 for the weight-paging per-expert offset array (binding 6). - static constexpr uint32_t mul_mat_vec_id_num_bindings = 7; + if (vk_instance.devices[idx] == nullptr) { + VK_LOG_DEBUG("Initializing new vk_device"); + vk_device device = std::make_shared<vk_device_struct>(); + vk_instance.devices[idx] = device; -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) -#define OCP_DMMV_LEN(NAME, REDUC) (device->ocp_fp4 ? NAME ## _ocp_len[REDUC] : NAME ## _len[REDUC]) -#define OCP_DMMV_DATA(NAME, REDUC) (device->ocp_fp4 ? NAME ## _ocp_data[REDUC] : NAME ## _data[REDUC]) -#else -#define OCP_DMMV_LEN(NAME, REDUC) NAME ## _len[REDUC] -#define OCP_DMMV_DATA(NAME, REDUC) NAME ## _data[REDUC] -#endif + device->memory_logger = std::unique_ptr<vk_memory_logger>(new vk_memory_logger()); - for (uint32_t w = 0; w < DMMV_WG_SIZE_COUNT; ++w) { - const uint32_t wg_size_subgroup = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size : (subgroup_size * 4); - const uint32_t wg_size_subgroup16 = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size16 : (subgroup_size16 * 4); + size_t dev_num = vk_instance.device_indices[idx]; - const shader_reduction_mode reduc = (use_subgroups && w == DMMV_WG_SIZE_SUBGROUP) ? SHADER_REDUCTION_MODE_SUBGROUP : - (use_subgroups && w == DMMV_WG_SIZE_LARGE) ? SHADER_REDUCTION_MODE_HYBRID : - SHADER_REDUCTION_MODE_SHMEM; + std::vector<vk::PhysicalDevice> physical_devices = vk_instance.instance.enumeratePhysicalDevices(); - const shader_reduction_mode reduc16 = (use_subgroups16 && w == DMMV_WG_SIZE_SUBGROUP) ? SHADER_REDUCTION_MODE_SUBGROUP : - (use_subgroups16 && w == DMMV_WG_SIZE_LARGE) ? SHADER_REDUCTION_MODE_HYBRID : - SHADER_REDUCTION_MODE_SHMEM; + if (dev_num >= physical_devices.size()) { + std::cerr << "ggml_vulkan: Device with index " << dev_num << " does not exist." << std::endl; + throw std::runtime_error("Device not found"); + } - for (uint32_t i = 0; i < device->mul_mat_vec_max_cols_eff; ++i) { - // Rows per workgroup for BF16, as a function of the batch width. - // After the rows-outside restructure the shader holds NUM_COLS B - // vec4s plus NUM_COLS*NUM_ROWS accumulators live at once, so the - // register footprint grows with the product of the two. Cap that - // product so wide batches do not spill and crater occupancy: on - // GCN this yields 4 rows up to 8 columns, tapering to 2 at 16. - // Where rm_bf16 is the stock 2 (everything except GCN) this clamp - // never binds, so those pipelines are unchanged. - const uint32_t rm_bf16_i = std::max(1u, std::min(rm_bf16, 32u / (i + 1))); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f32_f32", arr_dmmv_f32_f32_f32_len[reduc], arr_dmmv_f32_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1, 1, 1}, {wg_size_subgroup, 1, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f32_f32", arr_dmmv_f16_f32_f32_len[reduc], arr_dmmv_f16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f32_f32", arr_dmmv_bf16_f32_f32_len[reduc], arr_dmmv_bf16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_bf16_i, 1, 1}, {wg_size_subgroup, rm_bf16_i, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f32_f32", arr_dmmv_q1_0_f32_f32_len[reduc], arr_dmmv_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f32_f32", arr_dmmv_q2_0_f32_f32_len[reduc], arr_dmmv_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f32_f32", arr_dmmv_q4_0_f32_f32_len[reduc], arr_dmmv_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f32_f32", arr_dmmv_q4_1_f32_f32_len[reduc], arr_dmmv_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f32_f32", arr_dmmv_q5_0_f32_f32_len[reduc], arr_dmmv_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f32_f32", arr_dmmv_q5_1_f32_f32_len[reduc], arr_dmmv_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f32_f32", arr_dmmv_q8_0_f32_f32_len[reduc], arr_dmmv_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f32_f32", arr_dmmv_q2_k_f32_f32_len[reduc16], arr_dmmv_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f32_f32", arr_dmmv_tq2_0_f32_f32_len[reduc16], arr_dmmv_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f32_f32", arr_dmmv_q3_k_f32_f32_len[reduc16], arr_dmmv_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f32_f32", arr_dmmv_q4_k_f32_f32_len[reduc16], arr_dmmv_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f32_f32", arr_dmmv_q5_k_f32_f32_len[reduc16], arr_dmmv_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f32_f32", arr_dmmv_q6_k_f32_f32_len[reduc16], arr_dmmv_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f32_f32", arr_dmmv_iq1_s_f32_f32_len[reduc16], arr_dmmv_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f32_f32", arr_dmmv_iq1_m_f32_f32_len[reduc16], arr_dmmv_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f32_f32", arr_dmmv_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f32_f32", arr_dmmv_iq2_xs_f32_f32_len[reduc16], arr_dmmv_iq2_xs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f32_f32", arr_dmmv_iq2_s_f32_f32_len[reduc16], arr_dmmv_iq2_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f32_f32", arr_dmmv_iq3_xxs_f32_f32_len[reduc16], arr_dmmv_iq3_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f32_f32", arr_dmmv_iq3_s_f32_f32_len[reduc16], arr_dmmv_iq3_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f32_f32", arr_dmmv_iq4_xs_f32_f32_len[reduc16], arr_dmmv_iq4_xs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f32_f32", arr_dmmv_iq4_nl_f32_f32_len[reduc16], arr_dmmv_iq4_nl_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f32_f32", OCP_DMMV_LEN(arr_dmmv_mxfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_mxfp4_f32_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f32_f32", OCP_DMMV_LEN(arr_dmmv_nvfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_nvfp4_f32_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + device->physical_device = physical_devices[dev_num]; + const std::vector<vk::ExtensionProperties> ext_props = device->physical_device.enumerateDeviceExtensionProperties(); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f16_f32", arr_dmmv_f32_f16_f32_len[reduc], arr_dmmv_f32_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1, 1, 1}, {wg_size_subgroup, 1, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f16_f32", arr_dmmv_f16_f16_f32_len[reduc], arr_dmmv_f16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f16_f32", arr_dmmv_bf16_f16_f32_len[reduc], arr_dmmv_bf16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_bf16_i, 1, 1}, {wg_size_subgroup, rm_bf16_i, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f16_f32", arr_dmmv_q1_0_f16_f32_len[reduc], arr_dmmv_q1_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f16_f32", arr_dmmv_q2_0_f16_f32_len[reduc], arr_dmmv_q2_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f16_f32", arr_dmmv_q4_0_f16_f32_len[reduc], arr_dmmv_q4_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f16_f32", arr_dmmv_q4_1_f16_f32_len[reduc], arr_dmmv_q4_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f16_f32", arr_dmmv_q5_0_f16_f32_len[reduc], arr_dmmv_q5_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f16_f32", arr_dmmv_q5_1_f16_f32_len[reduc], arr_dmmv_q5_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f16_f32", arr_dmmv_q8_0_f16_f32_len[reduc], arr_dmmv_q8_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f16_f32", arr_dmmv_q2_k_f16_f32_len[reduc16], arr_dmmv_q2_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f16_f32", arr_dmmv_tq2_0_f16_f32_len[reduc16], arr_dmmv_tq2_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f16_f32", arr_dmmv_q3_k_f16_f32_len[reduc16], arr_dmmv_q3_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f16_f32", arr_dmmv_q4_k_f16_f32_len[reduc16], arr_dmmv_q4_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f16_f32", arr_dmmv_q5_k_f16_f32_len[reduc16], arr_dmmv_q5_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f16_f32", arr_dmmv_q6_k_f16_f32_len[reduc16], arr_dmmv_q6_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f16_f32", arr_dmmv_iq1_s_f16_f32_len[reduc16], arr_dmmv_iq1_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f16_f32", arr_dmmv_iq1_m_f16_f32_len[reduc16], arr_dmmv_iq1_m_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f16_f32", arr_dmmv_iq2_xxs_f16_f32_len[reduc16], arr_dmmv_iq2_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f16_f32", arr_dmmv_iq2_xs_f16_f32_len[reduc16], arr_dmmv_iq2_xs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f16_f32", arr_dmmv_iq2_s_f16_f32_len[reduc16], arr_dmmv_iq2_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f16_f32", arr_dmmv_iq3_xxs_f16_f32_len[reduc16], arr_dmmv_iq3_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f16_f32", arr_dmmv_iq3_s_f16_f32_len[reduc16], arr_dmmv_iq3_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f16_f32", arr_dmmv_iq4_xs_f16_f32_len[reduc16], arr_dmmv_iq4_xs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f16_f32", arr_dmmv_iq4_nl_f16_f32_len[reduc16], arr_dmmv_iq4_nl_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f16_f32", OCP_DMMV_LEN(arr_dmmv_mxfp4_f16_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_mxfp4_f16_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f16_f32", OCP_DMMV_LEN(arr_dmmv_nvfp4_f16_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_nvfp4_f16_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + device->architecture = get_device_architecture(device->physical_device); -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - const uint32_t subgroup_size_int = (device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control) ? device->subgroup_min_size : device->subgroup_size; - const uint32_t wg_size_subgroup_int = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size_int : (subgroup_size_int * 4); + const char* GGML_VK_PREFER_HOST_MEMORY = getenv("GGML_VK_PREFER_HOST_MEMORY"); + device->prefer_host_memory = GGML_VK_PREFER_HOST_MEMORY != nullptr; - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_q8_1_f32", arr_dmmv_q2_0_q8_1_f32_len[reduc], arr_dmmv_q2_0_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(2*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(2*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_q8_1_f32", arr_dmmv_q4_0_q8_1_f32_len[reduc], arr_dmmv_q4_0_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_q8_1_f32", arr_dmmv_q4_1_q8_1_f32_len[reduc], arr_dmmv_q4_1_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_q8_1_f32", arr_dmmv_q5_0_q8_1_f32_len[reduc], arr_dmmv_q5_0_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_q8_1_f32", arr_dmmv_q5_1_q8_1_f32_len[reduc], arr_dmmv_q5_1_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_q8_1_f32", arr_dmmv_q8_0_q8_1_f32_len[reduc], arr_dmmv_q8_0_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + const char* GGML_VK_DISABLE_HOST_VISIBLE_VIDMEM = getenv("GGML_VK_DISABLE_HOST_VISIBLE_VIDMEM"); + device->disable_host_visible_vidmem = GGML_VK_DISABLE_HOST_VISIBLE_VIDMEM != nullptr; - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_q8_1_f32", arr_dmmv_mxfp4_q8_1_f32_len[reduc], arr_dmmv_mxfp4_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(2*rm_stdq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(2*rm_stdq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + const char* GGML_VK_ALLOW_SYSMEM_FALLBACK = getenv("GGML_VK_ALLOW_SYSMEM_FALLBACK"); + device->allow_sysmem_fallback = GGML_VK_ALLOW_SYSMEM_FALLBACK != nullptr; - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_q8_1_f32", arr_dmmv_q2_k_q8_1_f32_len[reduc], arr_dmmv_q2_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(2*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(2*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_q8_1_f32", arr_dmmv_q3_k_q8_1_f32_len[reduc], arr_dmmv_q3_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_q8_1_f32", arr_dmmv_q4_k_q8_1_f32_len[reduc], arr_dmmv_q4_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_q8_1_f32", arr_dmmv_q5_k_q8_1_f32_len[reduc], arr_dmmv_q5_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_q8_1_f32", arr_dmmv_q6_k_q8_1_f32_len[reduc], arr_dmmv_q6_k_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_int_n(1*rm_kq_int, i), 1, 1}, {wg_size_subgroup_int, rm_int_n(1*rm_kq_int, i), i+1}, 1, true, use_subgroups, subgroup_size_int); + const char* GGML_VK_DISABLE_GRAPH_OPTIMIZE = getenv("GGML_VK_DISABLE_GRAPH_OPTIMIZE"); + device->disable_graph_optimize = GGML_VK_DISABLE_GRAPH_OPTIMIZE != nullptr; - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_q8_1_f32", arr_dmmv_iq1_s_q8_1_f32_len[reduc], arr_dmmv_iq1_s_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_iq_int(i), 1, 1}, {wg_size_subgroup_int, 1*rm_iq_int(i), i+1}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_q8_1_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_q8_1_f32", arr_dmmv_iq1_m_q8_1_f32_len[reduc], arr_dmmv_iq1_m_q8_1_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_iq_int(i), 1, 1}, {wg_size_subgroup_int, 1*rm_iq_int(i), i+1}, 1, true, use_subgroups, subgroup_size_int); + bool fp16_storage = false; + bool fp16_compute = false; + bool maintenance4_support = false; + bool sm_builtins = false; + bool amd_shader_core_properties2 = false; + bool pipeline_robustness = false; + bool coopmat2_support = false; + bool coopmat2_decode_vector_support = false; + bool pipeline_executable_properties_support = false; + bool internally_sync_support = false; + device->coopmat_support = false; + device->integer_dot_product = false; + device->shader_64b_indexing = false; + bool bfloat16_support = false; + bool dot2_f16_support = false; + bool ocp_microscaling_extension = false; + bool shader_float8_extension = false; + for (const auto& properties : ext_props) { + if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) { + maintenance4_support = true; + } else if (strcmp("VK_KHR_16bit_storage", properties.extensionName) == 0) { + fp16_storage = true; + } else if (strcmp("VK_KHR_shader_float16_int8", properties.extensionName) == 0) { + fp16_compute = true; + } else if (strcmp("VK_NV_shader_sm_builtins", properties.extensionName) == 0) { + sm_builtins = true; + } else if (strcmp("VK_AMD_shader_core_properties2", properties.extensionName) == 0) { + amd_shader_core_properties2 = true; + } else if (strcmp("VK_EXT_pipeline_robustness", properties.extensionName) == 0) { + pipeline_robustness = true; + } else if (strcmp("VK_EXT_subgroup_size_control", properties.extensionName) == 0) { + device->subgroup_size_control = true; +#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + } else if (strcmp("VK_KHR_cooperative_matrix", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_COOPMAT")) { + device->coopmat_support = true; + device->coopmat_m = 0; + device->coopmat_n = 0; + device->coopmat_k = 0; +#endif +#if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) + } else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_COOPMAT2")) { + coopmat2_support = true; +#endif + } else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) { + coopmat2_decode_vector_support = true; +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + } else if (strcmp("VK_KHR_shader_integer_dot_product", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_INTEGER_DOT_PRODUCT")) { + device->integer_dot_product = true; +#endif +#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + } else if (strcmp("VK_KHR_shader_bfloat16", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_BFLOAT16")) { + bfloat16_support = true; +#endif +#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) + } else if (strcmp(VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME, properties.extensionName) == 0) { + ocp_microscaling_extension = true; +#endif +#if defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) + } else if (strcmp(VK_EXT_SHADER_FLOAT8_EXTENSION_NAME, properties.extensionName) == 0) { + shader_float8_extension = true; +#endif + } else if (strcmp("VK_VALVE_shader_mixed_float_dot_product", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_DOT2")) { + dot2_f16_support = true; + } else if (strcmp("VK_KHR_pipeline_executable_properties", properties.extensionName) == 0) { + pipeline_executable_properties_support = true; + } else if (strcmp("VK_EXT_memory_priority", properties.extensionName) == 0 && + getenv("GGML_VK_ENABLE_MEMORY_PRIORITY")) { + device->memory_priority = true; + } else if (strcmp("VK_EXT_external_memory_host", properties.extensionName) == 0) { + device->external_memory_host = true; + } else if (strcmp("VK_KHR_external_semaphore_fd", properties.extensionName) == 0) { + device->external_semaphore_fd = true; +#if defined(VK_EXT_shader_64bit_indexing) + } else if (strcmp("VK_EXT_shader_64bit_indexing", properties.extensionName) == 0) { + device->shader_64b_indexing = true; +#endif + } else if (strcmp(VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME, properties.extensionName) == 0) { + internally_sync_support = true; + } else if (strcmp("VK_EXT_device_fault", properties.extensionName) == 0) { + device->device_fault = true; } -#endif // GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT } - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F32 ], "mul_mat_vec_id_f32_f32", arr_dmmv_id_f32_f32_f32_len[reduc], arr_dmmv_id_f32_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1, 1, 1}, {wg_size_subgroup, 1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F16 ], "mul_mat_vec_id_f16_f32", arr_dmmv_id_f16_f32_f32_len[reduc], arr_dmmv_id_f16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_BF16], "mul_mat_vec_id_bf16_f32", arr_dmmv_id_bf16_f32_f32_len[reduc], arr_dmmv_id_bf16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q1_0], "mul_mat_vec_id_q1_0_f32", arr_dmmv_id_q1_0_f32_f32_len[reduc], arr_dmmv_id_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_0], "mul_mat_vec_id_q2_0_f32", arr_dmmv_id_q2_0_f32_f32_len[reduc], arr_dmmv_id_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_0], "mul_mat_vec_id_q4_0_f32", arr_dmmv_id_q4_0_f32_f32_len[reduc], arr_dmmv_id_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_1], "mul_mat_vec_id_q4_1_f32", arr_dmmv_id_q4_1_f32_f32_len[reduc], arr_dmmv_id_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_0], "mul_mat_vec_id_q5_0_f32", arr_dmmv_id_q5_0_f32_f32_len[reduc], arr_dmmv_id_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_1], "mul_mat_vec_id_q5_1_f32", arr_dmmv_id_q5_1_f32_f32_len[reduc], arr_dmmv_id_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q8_0], "mul_mat_vec_id_q8_0_f32", arr_dmmv_id_q8_0_f32_f32_len[reduc], arr_dmmv_id_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_K], "mul_mat_vec_id_q2_k_f32", arr_dmmv_id_q2_k_f32_f32_len[reduc16], arr_dmmv_id_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ2_0], "mul_mat_vec_id_tq2_0_f32", arr_dmmv_id_tq2_0_f32_f32_len[reduc16], arr_dmmv_id_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q3_K], "mul_mat_vec_id_q3_k_f32", arr_dmmv_id_q3_k_f32_f32_len[reduc16], arr_dmmv_id_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_K], "mul_mat_vec_id_q4_k_f32", arr_dmmv_id_q4_k_f32_f32_len[reduc16], arr_dmmv_id_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_K], "mul_mat_vec_id_q5_k_f32", arr_dmmv_id_q5_k_f32_f32_len[reduc16], arr_dmmv_id_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q6_K], "mul_mat_vec_id_q6_k_f32", arr_dmmv_id_q6_k_f32_f32_len[reduc16], arr_dmmv_id_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_f32", arr_dmmv_id_iq1_s_f32_f32_len[reduc16], arr_dmmv_id_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_f32", arr_dmmv_id_iq1_m_f32_f32_len[reduc16], arr_dmmv_id_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XXS], "mul_mat_vec_id_iq2_xxs_f32", arr_dmmv_id_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XS], "mul_mat_vec_id_iq2_xs_f32", arr_dmmv_id_iq2_xs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_S], "mul_mat_vec_id_iq2_s_f32", arr_dmmv_id_iq2_s_f32_f32_len[reduc16], arr_dmmv_id_iq2_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_XXS], "mul_mat_vec_id_iq3_xxs_f32", arr_dmmv_id_iq3_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq3_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_S], "mul_mat_vec_id_iq3_s_f32", arr_dmmv_id_iq3_s_f32_f32_len[reduc16], arr_dmmv_id_iq3_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_XS], "mul_mat_vec_id_iq4_xs_f32", arr_dmmv_id_iq4_xs_f32_f32_len[reduc16], arr_dmmv_id_iq4_xs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_NL], "mul_mat_vec_id_iq4_nl_f32", arr_dmmv_id_iq4_nl_f32_f32_len[reduc16], arr_dmmv_id_iq4_nl_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_MXFP4], "mul_mat_vec_id_mxfp4_f32", OCP_DMMV_LEN(arr_dmmv_id_mxfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_id_mxfp4_f32_f32, reduc16), "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_NVFP4], "mul_mat_vec_id_nvfp4_f32", OCP_DMMV_LEN(arr_dmmv_id_nvfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_id_nvfp4_f32_f32, reduc16), "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + vk::PhysicalDeviceProperties2 props2; + vk::PhysicalDeviceMaintenance3Properties props3; + vk::PhysicalDeviceMaintenance4Properties props4; + vk::PhysicalDeviceSubgroupProperties subgroup_props; + vk::PhysicalDeviceDriverProperties driver_props; + vk::PhysicalDeviceShaderSMBuiltinsPropertiesNV sm_props; + vk::PhysicalDeviceShaderCoreProperties2AMD amd_shader_core_properties2_props; + vk::PhysicalDeviceVulkan11Properties vk11_props; + vk::PhysicalDeviceVulkan12Properties vk12_props; + vk::PhysicalDeviceSubgroupSizeControlPropertiesEXT subgroup_size_control_props; + vk::PhysicalDeviceShaderIntegerDotProductPropertiesKHR shader_integer_dot_product_props; + vk::PhysicalDeviceExternalMemoryHostPropertiesEXT external_memory_host_props; -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - const uint32_t subgroup_size_int = (device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control) ? device->subgroup_min_size : device->subgroup_size; - const uint32_t wg_size_subgroup_int = (w == DMMV_WG_SIZE_SUBGROUP) ? subgroup_size_int : (subgroup_size_int * 4); + props2.pNext = &props3; + props3.pNext = &subgroup_props; + subgroup_props.pNext = &driver_props; + driver_props.pNext = &vk11_props; + vk11_props.pNext = &vk12_props; - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q2_0], "mul_mat_vec_id_q2_0_q8_1_f32", arr_dmmv_id_q2_0_q8_1_f32_len[reduc], arr_dmmv_id_q2_0_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(2*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(2*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q4_0], "mul_mat_vec_id_q4_0_q8_1_f32", arr_dmmv_id_q4_0_q8_1_f32_len[reduc], arr_dmmv_id_q4_0_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q4_1], "mul_mat_vec_id_q4_1_q8_1_f32", arr_dmmv_id_q4_1_q8_1_f32_len[reduc], arr_dmmv_id_q4_1_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q5_0], "mul_mat_vec_id_q5_0_q8_1_f32", arr_dmmv_id_q5_0_q8_1_f32_len[reduc], arr_dmmv_id_q5_0_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q5_1], "mul_mat_vec_id_q5_1_q8_1_f32", arr_dmmv_id_q5_1_q8_1_f32_len[reduc], arr_dmmv_id_q5_1_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q8_0], "mul_mat_vec_id_q8_0_q8_1_f32", arr_dmmv_id_q8_0_q8_1_f32_len[reduc], arr_dmmv_id_q8_0_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); + VkBaseOutStructure * last_struct = (VkBaseOutStructure *)&vk12_props; - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_MXFP4], "mul_mat_vec_id_mxfp4_q8_1_f32", arr_dmmv_id_mxfp4_q8_1_f32_len[reduc], arr_dmmv_id_mxfp4_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(2*rm_stdq_int), 1, 1}, {wg_size_subgroup_int, rm_id(2*rm_stdq_int)}, 1, true, use_subgroups, subgroup_size_int); + if (maintenance4_support) { + last_struct->pNext = (VkBaseOutStructure *)&props4; + last_struct = (VkBaseOutStructure *)&props4; + } + if (sm_builtins) { + last_struct->pNext = (VkBaseOutStructure *)&sm_props; + last_struct = (VkBaseOutStructure *)&sm_props; + } + if (amd_shader_core_properties2) { + last_struct->pNext = (VkBaseOutStructure *)&amd_shader_core_properties2_props; + last_struct = (VkBaseOutStructure *)&amd_shader_core_properties2_props; + } + if (device->subgroup_size_control) { + last_struct->pNext = (VkBaseOutStructure *)&subgroup_size_control_props; + last_struct = (VkBaseOutStructure *)&subgroup_size_control_props; + } - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q2_K], "mul_mat_vec_id_q2_k_q8_1_f32", arr_dmmv_id_q2_k_q8_1_f32_len[reduc], arr_dmmv_id_q2_k_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(2*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(2*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q3_K], "mul_mat_vec_id_q3_k_q8_1_f32", arr_dmmv_id_q3_k_q8_1_f32_len[reduc], arr_dmmv_id_q3_k_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q4_K], "mul_mat_vec_id_q4_k_q8_1_f32", arr_dmmv_id_q4_k_q8_1_f32_len[reduc], arr_dmmv_id_q4_k_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q5_K], "mul_mat_vec_id_q5_k_q8_1_f32", arr_dmmv_id_q5_k_q8_1_f32_len[reduc], arr_dmmv_id_q5_k_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_Q6_K], "mul_mat_vec_id_q6_k_q8_1_f32", arr_dmmv_id_q6_k_q8_1_f32_len[reduc], arr_dmmv_id_q6_k_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_id(1*rm_kq_int), 1, 1}, {wg_size_subgroup_int, rm_id(1*rm_kq_int)}, 1, true, use_subgroups, subgroup_size_int); +#if defined(VK_NV_cooperative_matrix2) + vk::PhysicalDeviceCooperativeMatrix2PropertiesNV coopmat2_props; + if (coopmat2_support) { + last_struct->pNext = (VkBaseOutStructure *)&coopmat2_props; + last_struct = (VkBaseOutStructure *)&coopmat2_props; + } +#endif - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_q8_1_f32", arr_dmmv_id_iq1_s_q8_1_f32_len[reduc], arr_dmmv_id_iq1_s_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*rm_iq_int(0), 1, 1}, {wg_size_subgroup_int, 1*rm_iq_int(0)}, 1, true, use_subgroups, subgroup_size_int); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[w][GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_q8_1_f32", arr_dmmv_id_iq1_m_q8_1_f32_len[reduc], arr_dmmv_id_iq1_m_q8_1_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*rm_iq_int(0), 1, 1}, {wg_size_subgroup_int, 1*rm_iq_int(0)}, 1, true, use_subgroups, subgroup_size_int); + if (device->integer_dot_product) { + last_struct->pNext = (VkBaseOutStructure *)&shader_integer_dot_product_props; + last_struct = (VkBaseOutStructure *)&shader_integer_dot_product_props; } -#endif // GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT - } -#undef OCP_DMMV_DATA -#undef OCP_DMMV_LEN + if (device->external_memory_host) { + last_struct->pNext = (VkBaseOutStructure *)&external_memory_host_props; + last_struct = (VkBaseOutStructure *)&external_memory_host_props; + } -#if !defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - GGML_UNUSED(rm_stdq_int); - GGML_UNUSED(rm_kq_int); - GGML_UNUSED(is_rdna3); - GGML_UNUSED(rm_int_n); - GGML_UNUSED(rm_id); - GGML_UNUSED(rm_iq_int); -#endif + device->physical_device.getProperties2(&props2); + device->properties = props2.properties; + device->vendor_id = device->properties.vendorID; + device->driver_id = driver_props.driverID; - // dequant shaders - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_F32 ], "f32_to_f16", dequant_f32_len, dequant_f32_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q1_0], "dequant_q1_0", dequant_q1_0_len, dequant_q1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 8, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q2_0], "dequant_q2_0", dequant_q2_0_len, dequant_q2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_0], "dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_1], "dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_0], "dequant_q5_0", dequant_q5_0_len, dequant_q5_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_1], "dequant_q5_1", dequant_q5_1_len, dequant_q5_1_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q8_0], "dequant_q8_0", dequant_q8_0_len, dequant_q8_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant_transpose[GGML_TYPE_Q8_0], "dequant_q8_0_transpose", dequant_q8_0_transpose_len, dequant_q8_0_transpose_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q2_K], "dequant_q2_k", dequant_q2_k_len, dequant_q2_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ2_0], "dequant_tq2_0", dequant_tq2_0_len, dequant_tq2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q3_K], "dequant_q3_k", dequant_q3_k_len, dequant_q3_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_K], "dequant_q4_k", dequant_q4_k_len, dequant_q4_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_K], "dequant_q5_k", dequant_q5_k_len, dequant_q5_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q6_K], "dequant_q6_k", dequant_q6_k_len, dequant_q6_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ1_S], "dequant_iq1_s", dequant_iq1_s_len, dequant_iq1_s_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ1_M], "dequant_iq1_m", dequant_iq1_m_len, dequant_iq1_m_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ2_XXS], "dequant_iq2_xxs", dequant_iq2_xxs_len, dequant_iq2_xxs_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ2_XS], "dequant_iq2_xs", dequant_iq2_xs_len, dequant_iq2_xs_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ2_S], "dequant_iq2_s", dequant_iq2_s_len, dequant_iq2_s_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ3_XXS], "dequant_iq3_xxs", dequant_iq3_xxs_len, dequant_iq3_xxs_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ3_S], "dequant_iq3_s", dequant_iq3_s_len, dequant_iq3_s_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ4_XS], "dequant_iq4_xs", dequant_iq4_xs_len, dequant_iq4_xs_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ4_NL], "dequant_iq4_nl", dequant_iq4_nl_len, dequant_iq4_nl_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_MXFP4], "dequant_mxfp4", dequant_mxfp4_len, dequant_mxfp4_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_NVFP4], "dequant_nvfp4", dequant_nvfp4_len, dequant_nvfp4_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + if (device->driver_id == vk::DriverId::eMoltenvk) { + // Disable external_memory_host until https://github.com/KhronosGroup/MoltenVK/pull/2622 + // is available in the Vulkan SDK. + device->external_memory_host = false; + } - // get_rows - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_F32 ], "get_rows_f32", get_rows_f32_len, get_rows_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_F16 ], "get_rows_f16", get_rows_f16_len, get_rows_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_BF16], "get_rows_bf16", get_rows_bf16_len, get_rows_bf16_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q1_0], "get_rows_q1_0", get_rows_q1_0_len, get_rows_q1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q2_0], "get_rows_q2_0", get_rows_q2_0_len, get_rows_q2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_0], "get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_1], "get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_0], "get_rows_q5_0", get_rows_q5_0_len, get_rows_q5_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_1], "get_rows_q5_1", get_rows_q5_1_len, get_rows_q5_1_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q8_0], "get_rows_q8_0", get_rows_q8_0_len, get_rows_q8_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q2_K], "get_rows_q2_k", get_rows_q2_k_len, get_rows_q2_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TQ2_0], "get_rows_tq2_0", get_rows_tq2_0_len, get_rows_tq2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q3_K], "get_rows_q3_k", get_rows_q3_k_len, get_rows_q3_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_K], "get_rows_q4_k", get_rows_q4_k_len, get_rows_q4_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_K], "get_rows_q5_k", get_rows_q5_k_len, get_rows_q5_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q6_K], "get_rows_q6_k", get_rows_q6_k_len, get_rows_q6_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ1_S], "get_rows_iq1_s", get_rows_iq1_s_len, get_rows_iq1_s_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ1_M], "get_rows_iq1_m", get_rows_iq1_m_len, get_rows_iq1_m_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ2_XXS], "get_rows_iq2_xxs", get_rows_iq2_xxs_len, get_rows_iq2_xxs_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ2_XS], "get_rows_iq2_xs", get_rows_iq2_xs_len, get_rows_iq2_xs_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ2_S], "get_rows_iq2_s", get_rows_iq2_s_len, get_rows_iq2_s_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ3_XXS], "get_rows_iq3_xxs", get_rows_iq3_xxs_len, get_rows_iq3_xxs_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ3_S], "get_rows_iq3_s", get_rows_iq3_s_len, get_rows_iq3_s_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ4_XS], "get_rows_iq4_xs", get_rows_iq4_xs_len, get_rows_iq4_xs_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ4_NL], "get_rows_iq4_nl", get_rows_iq4_nl_len, get_rows_iq4_nl_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_MXFP4], "get_rows_mxfp4", get_rows_mxfp4_len, get_rows_mxfp4_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_NVFP4], "get_rows_nvfp4", get_rows_nvfp4_len, get_rows_nvfp4_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_I32], "get_rows_i32", get_rows_i32_len, get_rows_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + // Implementing the async backend interfaces seems broken on older Intel HW, + // see https://github.com/ggml-org/llama.cpp/issues/17302. + device->support_async = (device->vendor_id != VK_VENDOR_ID_INTEL || + std::string(device->properties.deviceName.data()).find("(DG1)") == std::string::npos) && + getenv("GGML_VK_DISABLE_ASYNC") == nullptr; - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_F32 ], "get_rows_f32_f32", get_rows_f32_f32_len, get_rows_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_F16 ], "get_rows_f16_f32", get_rows_f16_f32_len, get_rows_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_BF16], "get_rows_bf16_f32", get_rows_bf16_f32_len, get_rows_bf16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q1_0], "get_rows_q1_0_f32", get_rows_q1_0_f32_len, get_rows_q1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q2_0], "get_rows_q2_0_f32", get_rows_q2_0_f32_len, get_rows_q2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_0], "get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_1], "get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_0], "get_rows_q5_0_f32", get_rows_q5_0_f32_len, get_rows_q5_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_1], "get_rows_q5_1_f32", get_rows_q5_1_f32_len, get_rows_q5_1_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q8_0], "get_rows_q8_0_f32", get_rows_q8_0_f32_len, get_rows_q8_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q2_K], "get_rows_q2_k_f32", get_rows_q2_k_f32_len, get_rows_q2_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TQ2_0], "get_rows_tq2_0_f32", get_rows_tq2_0_f32_len, get_rows_tq2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q3_K], "get_rows_q3_k_f32", get_rows_q3_k_f32_len, get_rows_q3_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_K], "get_rows_q4_k_f32", get_rows_q4_k_f32_len, get_rows_q4_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_K], "get_rows_q5_k_f32", get_rows_q5_k_f32_len, get_rows_q5_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q6_K], "get_rows_q6_k_f32", get_rows_q6_k_f32_len, get_rows_q6_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ1_S], "get_rows_iq1_s_f32", get_rows_iq1_s_f32_len, get_rows_iq1_s_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ1_M], "get_rows_iq1_m_f32", get_rows_iq1_m_f32_len, get_rows_iq1_m_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ2_XXS], "get_rows_iq2_xxs_f32", get_rows_iq2_xxs_f32_len, get_rows_iq2_xxs_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ2_XS], "get_rows_iq2_xs_f32", get_rows_iq2_xs_f32_len, get_rows_iq2_xs_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ2_S], "get_rows_iq2_s_f32", get_rows_iq2_s_f32_len, get_rows_iq2_s_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ3_XXS], "get_rows_iq3_xxs_f32", get_rows_iq3_xxs_f32_len, get_rows_iq3_xxs_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ3_S], "get_rows_iq3_s_f32", get_rows_iq3_s_f32_len, get_rows_iq3_s_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ4_XS], "get_rows_iq4_xs_f32", get_rows_iq4_xs_f32_len, get_rows_iq4_xs_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ4_NL], "get_rows_iq4_nl_f32", get_rows_iq4_nl_f32_len, get_rows_iq4_nl_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_MXFP4], "get_rows_mxfp4_f32", get_rows_mxfp4_f32_len, get_rows_mxfp4_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_NVFP4], "get_rows_nvfp4_f32", get_rows_nvfp4_f32_len, get_rows_nvfp4_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_back_f32, "get_rows_back_f32", get_rows_back_f32_len, get_rows_back_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {256, 1, 1}, {}, 1, true); + if (!device->support_async) { + GGML_LOG_DEBUG("ggml_vulkan: WARNING: Async execution disabled on certain Intel devices.\n"); + } - ggml_vk_create_pipeline(device, device->pipeline_matmul_split_k_reduce, "split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_flash_attn_split_k_reduce, "fa_split_k_reduce", fa_split_k_reduce_len, fa_split_k_reduce_data, "main", 3, sizeof(vk_op_flash_attn_split_k_reduce_push_constants), {1, device->subgroup_size, 1}, {device->subgroup_size}, 1, true); + const char* GGML_VK_FORCE_MAX_ALLOCATION_SIZE = getenv("GGML_VK_FORCE_MAX_ALLOCATION_SIZE"); - for (auto &it : device->pipeline_fa_mask_opt) { - auto BrBc = it.first; - ggml_vk_create_pipeline(device, it.second, "fa_mask_opt", fa_mask_opt_len, fa_mask_opt_data, "main", 2, sizeof(vk_op_flash_attn_mask_opt_push_constants), {1, 1, 1}, {128, 128 / device->subgroup_size, BrBc.first, BrBc.second}, 1, true, true, device->subgroup_size); - } + if (GGML_VK_FORCE_MAX_ALLOCATION_SIZE != nullptr) { + device->max_memory_allocation_size = std::stoull(GGML_VK_FORCE_MAX_ALLOCATION_SIZE); + } else if (maintenance4_support) { + device->max_memory_allocation_size = std::min(props3.maxMemoryAllocationSize, props4.maxBufferSize); + } else { + device->max_memory_allocation_size = props3.maxMemoryAllocationSize; + } - if (device->subgroup_clustered && device->subgroup_require_full_support) { - ggml_vk_create_pipeline(device, device->pipeline_quantize_q8_1_x4, "quantize_q8_1_x4", quantize_q8_1_x4_subgroup_len, quantize_q8_1_x4_subgroup_data, "main", 2, sizeof(vk_quantize_q8_1_push_constants), {32 * device->subgroup_size / 8, 1, 1}, { device->subgroup_size }, 1, true, true); - } else { - ggml_vk_create_pipeline(device, device->pipeline_quantize_q8_1_x4, "quantize_q8_1_x4", quantize_q8_1_x4_len, quantize_q8_1_x4_data, "main", 2, sizeof(vk_quantize_q8_1_push_constants), {32 * device->subgroup_size / 8, 1, 1}, { device->subgroup_size }, 1); - } + const char* GGML_VK_FORCE_MAX_BUFFER_SIZE = getenv("GGML_VK_FORCE_MAX_BUFFER_SIZE"); - for (uint32_t i = 0; i < p021_max_gqa_ratio; ++i) { - if (device->subgroup_arithmetic && device->subgroup_require_full_support) { - ggml_vk_create_pipeline2(device, device->pipeline_mul_mat_vec_p021_f16_f32[i], "mul_mat_vec_p021_f16_f32"+std::to_string(i+1), mul_mat_vec_p021_f16_f32_subgroup_add_len, mul_mat_vec_p021_f16_f32_subgroup_add_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_p021_push_constants), {1, 1, 1}, {device->subgroup_size, i + 1}, 1, true, true); + if (GGML_VK_FORCE_MAX_BUFFER_SIZE != nullptr) { + device->max_buffer_size = std::stoull(GGML_VK_FORCE_MAX_BUFFER_SIZE); + } else if (maintenance4_support) { + device->max_buffer_size = props4.maxBufferSize; } else { - ggml_vk_create_pipeline2(device, device->pipeline_mul_mat_vec_p021_f16_f32[i], "mul_mat_vec_p021_f16_f32"+std::to_string(i+1), mul_mat_vec_p021_f16_f32_len, mul_mat_vec_p021_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_p021_push_constants), {1, 1, 1}, {device->subgroup_size, i + 1}, 1, true); + device->max_buffer_size = device->max_memory_allocation_size; } - } - ggml_vk_create_pipeline(device, device->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_nc_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_turbo_wht_f32, "turbo_wht_f32", turbo_wht_f32_len, turbo_wht_f32_data, "main", 3, sizeof(vk_op_turbo_wht_push_constants), {1, 1, 1}, {}, 1); - - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_f32, "rms_norm_mul_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_partials_f32, "rms_norm_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_partials_f32, "rms_norm_mul_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true); + const char* GGML_VK_SUBALLOCATION_BLOCK_SIZE = getenv("GGML_VK_SUBALLOCATION_BLOCK_SIZE"); - if (sizeof(vk_op_rms_norm_mul_rope_push_constants) <= device->properties.limits.maxPushConstantsSize) { - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f32, "rms_norm_mul_rope_f32_f32", rms_norm_mul_rope_f32_f32_len, rms_norm_mul_rope_f32_f32_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f16, "rms_norm_mul_rope_f32_f16", rms_norm_mul_rope_f32_f16_len, rms_norm_mul_rope_f32_f16_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true); - } + if (GGML_VK_SUBALLOCATION_BLOCK_SIZE != nullptr) { + device->suballocation_block_size = std::stoull(GGML_VK_SUBALLOCATION_BLOCK_SIZE); + } else { + // Limit batching of allocations to 1GB by default to avoid fragmentation issues + device->suballocation_block_size = 1024*1024*1024; + } + device->suballocation_block_size = std::min(device->suballocation_block_size, device->max_memory_allocation_size); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_sinkhorn_norm_f32[0], "sinkhorn_norm_f32_n2", sinkhorn_norm_f32_n2_len, sinkhorn_norm_f32_n2_data, "main", 2, sizeof(vk_op_sinkhorn_norm_push_constants), {64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_sinkhorn_norm_f32[1], "sinkhorn_norm_f32_n4", sinkhorn_norm_f32_n4_len, sinkhorn_norm_f32_n4_data, "main", 2, sizeof(vk_op_sinkhorn_norm_push_constants), {64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_sinkhorn_norm_f32[2], "sinkhorn_norm_f32_n8", sinkhorn_norm_f32_n8_len, sinkhorn_norm_f32_n8_data, "main", 2, sizeof(vk_op_sinkhorn_norm_push_constants), {64, 1, 1}, {}, 1); + device->subgroup_size = subgroup_props.subgroupSize; + device->subgroup_size_log2 = uint32_t(log2f(float(device->subgroup_size))); + device->uma = device->properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; + if (sm_builtins) { + device->shader_core_count = sm_props.shaderSMCount; + } else if (amd_shader_core_properties2) { + device->shader_core_count = amd_shader_core_properties2_props.activeComputeUnitCount; + } else if (device->vendor_id == VK_VENDOR_ID_INTEL) { + device->shader_core_count = ggml_vk_intel_shader_core_count(device->physical_device); + } else { + device->shader_core_count = 0; + } + device->float_controls_rte_fp16 = vk12_props.shaderRoundingModeRTEFloat16; + device->float_controls_denorm_preserve_fp16 = vk12_props.shaderDenormPreserveFloat16; - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f32, "cpy_f32_f32", cpy_f32_f32_len, cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f16_f16, "cpy_f16_f16", cpy_f16_f16_len, cpy_f16_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f16_f32, "cpy_f16_f32", cpy_f16_f32_len, cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_bf16,"cpy_f32_bf16",cpy_f32_bf16_len,cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_bf16_f32,"cpy_bf16_f32",cpy_bf16_f32_len,cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_i32_f32, "cpy_i32_f32", cpy_i32_f32_len, cpy_i32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_i32, "cpy_f32_i32", cpy_f32_i32_len, cpy_f32_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + device->subgroup_basic = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && + (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eBasic); + device->subgroup_arithmetic = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && + (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eArithmetic); +#ifdef __APPLE__ + // Workaround for subgroup arithmetic failing on MoltenVK with AMD GPUs (issue 15846) + if (device->vendor_id == VK_VENDOR_ID_AMD) { + device->subgroup_arithmetic = false; + } +#endif + device->subgroup_shuffle = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && + (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eShuffle); +#ifdef __APPLE__ + if (device->vendor_id == VK_VENDOR_ID_AMD) { + device->subgroup_shuffle = false; + } +#endif + device->subgroup_clustered = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && + (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eClustered); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_f32, "contig_cpy_f32_f32", contig_cpy_f32_f32_len, contig_cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_f16, "contig_cpy_f32_f16", contig_cpy_f32_f16_len, contig_cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f16_f16, "contig_cpy_f16_f16", contig_cpy_f16_f16_len, contig_cpy_f16_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f16_f32, "contig_cpy_f16_f32", contig_cpy_f16_f32_len, contig_cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_bf16,"contig_cpy_f32_bf16",contig_cpy_f32_bf16_len,contig_cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_bf16_f32,"contig_cpy_bf16_f32",contig_cpy_bf16_f32_len,contig_cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_i32_f32, "contig_cpy_i32_f32", contig_cpy_i32_f32_len, contig_cpy_i32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_i32, "contig_cpy_f32_i32", contig_cpy_f32_i32_len, contig_cpy_f32_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + device->subgroup_ballot = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && + (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eBallot); - ggml_vk_create_pipeline(device, device->pipeline_cpy_transpose_32, "cpy_transpose_32", cpy_transpose_32_len, cpy_transpose_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_transpose_16, "cpy_transpose_16", cpy_transpose_16_len, cpy_transpose_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_transpose_02_32, "cpy_transpose_02_32", cpy_transpose_02_32_len, cpy_transpose_02_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_transpose_02_16, "cpy_transpose_02_16", cpy_transpose_02_16_len, cpy_transpose_02_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); + device->subgroup_vote = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && + (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eVote); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q1_0], "cpy_f32_q1_0", cpy_f32_q1_0_len, cpy_f32_q1_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q2_0], "cpy_f32_q2_0", cpy_f32_q2_0_len, cpy_f32_q2_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q4_0], "cpy_f32_q4_0", cpy_f32_q4_0_len, cpy_f32_q4_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q4_1], "cpy_f32_q4_1", cpy_f32_q4_1_len, cpy_f32_q4_1_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q5_0], "cpy_f32_q5_0", cpy_f32_q5_0_len, cpy_f32_q5_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q5_1], "cpy_f32_q5_1", cpy_f32_q5_1_len, cpy_f32_q5_1_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q8_0], "cpy_f32_q8_0", cpy_f32_q8_0_len, cpy_f32_q8_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_IQ4_NL], "cpy_f32_iq4_nl", cpy_f32_iq4_nl_len, cpy_f32_iq4_nl_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); - // turbo4_0: 128 threads/workgroup, one workgroup per 128-element block; wg_denoms={1,1,1} - // so elements[0] (= n_blocks after CEIL_DIV by 128) maps 1-to-1 to workgroup count. - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_TURBO4_0], "cpy_f32_turbo4_0", cpy_f32_turbo4_0_len, cpy_f32_turbo4_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); + // Submit at least every 100 nodes, in case there are workloads without as much matmul. + device->max_nodes_per_submit = 100; + const char* GGML_VK_MAX_NODES_PER_SUBMIT = getenv("GGML_VK_MAX_NODES_PER_SUBMIT"); + if (GGML_VK_MAX_NODES_PER_SUBMIT != nullptr) { + uint32_t max_nodes_per_submit = std::stoul(GGML_VK_MAX_NODES_PER_SUBMIT); + device->max_nodes_per_submit = std::max(max_nodes_per_submit, 1u); + } -#define SET_ROWS(src_idx, src, itype) \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_F32], "set_rows_" #src "_f32" #itype, set_rows_ ## src ## _f32 ## itype ## _len, set_rows_ ## src ## _f32 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_F16], "set_rows_" #src "_f16" #itype, set_rows_ ## src ## _f16 ## itype ## _len, set_rows_ ## src ## _f16 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_BF16], "set_rows_" #src "_bf16" #itype, set_rows_ ## src ## _bf16 ## itype ## _len, set_rows_ ## src ## _bf16 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q1_0], "set_rows_" #src "_q1_0" #itype, set_rows_ ## src ## _q1_0 ## itype ## _len, set_rows_ ## src ## _q1_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q2_0], "set_rows_" #src "_q2_0" #itype, set_rows_ ## src ## _q2_0 ## itype ## _len, set_rows_ ## src ## _q2_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q4_0], "set_rows_" #src "_q4_0" #itype, set_rows_ ## src ## _q4_0 ## itype ## _len, set_rows_ ## src ## _q4_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q4_1], "set_rows_" #src "_q4_1" #itype, set_rows_ ## src ## _q4_1 ## itype ## _len, set_rows_ ## src ## _q4_1 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q5_0], "set_rows_" #src "_q5_0" #itype, set_rows_ ## src ## _q5_0 ## itype ## _len, set_rows_ ## src ## _q5_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q5_1], "set_rows_" #src "_q5_1" #itype, set_rows_ ## src ## _q5_1 ## itype ## _len, set_rows_ ## src ## _q5_1 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q8_0], "set_rows_" #src "_q8_0" #itype, set_rows_ ## src ## _q8_0 ## itype ## _len, set_rows_ ## src ## _q8_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_IQ4_NL], "set_rows_" #src "_iq4_nl" #itype, set_rows_ ## src ## _iq4_nl ## itype ## _len, set_rows_ ## src ## _iq4_nl ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); + const bool force_disable_f16 = getenv("GGML_VK_DISABLE_F16") != nullptr; - SET_ROWS(0, f32, _i32) - SET_ROWS(0, f32, _i64) - SET_ROWS(1, f16, _i32) - SET_ROWS(1, f16, _i64) + device->fp16 = !force_disable_f16 && fp16_storage && fp16_compute; - // fork: turbo4_0 SET_ROWS uses a bespoke cooperative shader (set_rows_f32_turbo4_0.comp) - // with a self-contained WHT, so it is not part of the templated SET_ROWS macro above and - // exists for an f32 source only -- register it directly in the src_idx 0 (fp32) slot. - ggml_vk_create_pipeline(device, device->pipeline_set_rows_i32[0][GGML_TYPE_TURBO4_0], "set_rows_turbo4_0_i32", set_rows_turbo4_0_i32_len, set_rows_turbo4_0_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_set_rows_i64[0][GGML_TYPE_TURBO4_0], "set_rows_turbo4_0_i64", set_rows_turbo4_0_i64_len, set_rows_turbo4_0_i64_data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); -#undef SET_ROWS + if (!ggml_vk_khr_cooperative_matrix_support(device->properties, driver_props, device->architecture)) { + device->coopmat_support = false; + } - // SP2 paged attention (GGML_OP_PAGED_ATTN_MT), F16 cache path (Task 3). - // Both shaders dispatch one workgroup per grid element (wg_denoms {1,1,1}), - // local_size_x = 128. scatter has 5 bindings, attention has 7. - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_F16], "paged_attn_scatter_f16", paged_attn_scatter_f16_len, paged_attn_scatter_f16_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_F16], "paged_attn_f16", paged_attn_f16_len, paged_attn_f16_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1, 1, 1}, {}, 1); + device->integer_dot_product = device->integer_dot_product && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated; - // turbo4_0 cache path (Task 4): no-RHT cooperative scatter quantizer + dequant load. - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_TURBO4_0], "paged_attn_scatter_turbo4_0", paged_attn_scatter_turbo4_0_len, paged_attn_scatter_turbo4_0_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_TURBO4_0], "paged_attn_turbo4_0", paged_attn_turbo4_0_len, paged_attn_turbo4_0_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1, 1, 1}, {}, 1); + device->min_imported_host_pointer_alignment = external_memory_host_props.minImportedHostPointerAlignment; - // turbo4_64 cache path (Task 3 read side + Task 4 scatter): dequant load - // plus the no-RHT cooperative scatter quantizer (64-element blocks). - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_TURBO4_64], "paged_attn_scatter_turbo4_64", paged_attn_scatter_turbo4_64_len, paged_attn_scatter_turbo4_64_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_Q8_0], "paged_attn_scatter_q8_0", paged_attn_scatter_q8_0_len, paged_attn_scatter_q8_0_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_TURBO4_64], "paged_attn_turbo4_64", paged_attn_turbo4_64_len, paged_attn_turbo4_64_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_Q8_0], "paged_attn_q8_0", paged_attn_q8_0_len, paged_attn_q8_0_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1,1,1}, {}, 1); + device->max_workgroup_size_log2 = uint32_t(log2f(float(device->properties.limits.maxComputeWorkGroupInvocations))); - // turbo4_64_ol cache path (SP2.5, 2026-07-01): turbo4_64 with 4 fixed-position - // outlier channels excluded from the group-norm/centroid quant, stored verbatim. - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_TURBO4_64_OL], "paged_attn_scatter_turbo4_64_ol", paged_attn_scatter_turbo4_64_ol_len, paged_attn_scatter_turbo4_64_ol_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_TURBO4_64_OL], "paged_attn_turbo4_64_ol", paged_attn_turbo4_64_ol_len, paged_attn_turbo4_64_ol_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1,1,1}, {}, 1); + std::vector<vk::QueueFamilyProperties> queue_family_props = device->physical_device.getQueueFamilyProperties(); - // turbo4_64_ol8 / turbo4_64_ol12 (2026-07-01 outlier-matrix sweep): same - // family as turbo4_64_ol above, just 8 and 12 fixed outlier channels - // respectively (see ggml-common.h TURBO4_64_OL8_OUTLIER_CHANNELS / - // TURBO4_64_OL12_OUTLIER_CHANNELS). Shader sources are shared with - // turbo4_64_ol via preprocessor defines (see vulkan-shaders-gen.cpp). - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_TURBO4_64_OL8], "paged_attn_scatter_turbo4_64_ol8", paged_attn_scatter_turbo4_64_ol8_len, paged_attn_scatter_turbo4_64_ol8_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_TURBO4_64_OL8], "paged_attn_turbo4_64_ol8", paged_attn_turbo4_64_ol8_len, paged_attn_turbo4_64_ol8_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_scatter[GGML_TYPE_TURBO4_64_OL12], "paged_attn_scatter_turbo4_64_ol12", paged_attn_scatter_turbo4_64_ol12_len, paged_attn_scatter_turbo4_64_ol12_data, "main", 5, sizeof(vk_op_paged_scatter_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn[GGML_TYPE_TURBO4_64_OL12], "paged_attn_turbo4_64_ol12", paged_attn_turbo4_64_ol12_len, paged_attn_turbo4_64_ol12_data, "main", 7, sizeof(vk_op_paged_attn_pc), {1,1,1}, {}, 1); + // Try to find a non-graphics compute queue and transfer-focused queues + // Allow overriding avoiding the graphics queue because it can increase performance on RADV + const bool allow_graphics_queue = (getenv("GGML_VK_ALLOW_GRAPHICS_QUEUE") != nullptr); + const vk::QueueFlagBits graphics_flag = allow_graphics_queue ? (vk::QueueFlagBits)0 : vk::QueueFlagBits::eGraphics; + const uint32_t compute_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eCompute, graphics_flag, -1, 1); + const uint32_t transfer_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eTransfer, vk::QueueFlagBits::eCompute | graphics_flag, compute_queue_family_index, 1); - // SP2 Task 5: split-K decode. Pass-1 (decode) has 7 bindings per cache type; - // pass-2 (reduce) is type-agnostic with 3 bindings. Both wg_denoms {1,1,1}, local_size 128. - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_F16], "paged_attn_decode_f16", paged_attn_decode_f16_len, paged_attn_decode_f16_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_TURBO4_0], "paged_attn_decode_turbo4_0", paged_attn_decode_turbo4_0_len, paged_attn_decode_turbo4_0_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_TURBO4_64], "paged_attn_decode_turbo4_64", paged_attn_decode_turbo4_64_len, paged_attn_decode_turbo4_64_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_TURBO4_64_OL], "paged_attn_decode_turbo4_64_ol", paged_attn_decode_turbo4_64_ol_len, paged_attn_decode_turbo4_64_ol_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_TURBO4_64_OL8], "paged_attn_decode_turbo4_64_ol8", paged_attn_decode_turbo4_64_ol8_len, paged_attn_decode_turbo4_64_ol8_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_TURBO4_64_OL12], "paged_attn_decode_turbo4_64_ol12", paged_attn_decode_turbo4_64_ol12_len, paged_attn_decode_turbo4_64_ol12_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode[GGML_TYPE_Q8_0], "paged_attn_decode_q8_0", paged_attn_decode_q8_0_len, paged_attn_decode_q8_0_data, "main", 7, sizeof(vk_op_paged_decode_pc), {1,1,1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_paged_attn_decode_reduce, "paged_attn_decode_reduce", paged_attn_decode_reduce_len, paged_attn_decode_reduce_data, "main", 3, sizeof(vk_op_paged_decode_reduce_pc), {1, 1, 1}, {}, 1); + const float priorities[] = { 1.0f, 1.0f }; + device->single_queue = compute_queue_family_index == transfer_queue_family_index && queue_family_props[compute_queue_family_index].queueCount == 1; + std::vector<vk::DeviceQueueCreateInfo> device_queue_create_infos; + vk::DeviceCreateInfo device_create_info{}; + std::vector<const char *> device_extensions; + vk::PhysicalDeviceFeatures device_features = device->physical_device.getFeatures(); - ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q1_0], "cpy_q1_0_f32", cpy_q1_0_f32_len, cpy_q1_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q1_0), 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q2_0], "cpy_q2_0_f32", cpy_q2_0_f32_len, cpy_q2_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q2_0), 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q4_0], "cpy_q4_0_f32", cpy_q4_0_f32_len, cpy_q4_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q4_0), 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q4_1], "cpy_q4_1_f32", cpy_q4_1_f32_len, cpy_q4_1_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q4_1), 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q5_0], "cpy_q5_0_f32", cpy_q5_0_f32_len, cpy_q5_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q5_0), 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q5_1], "cpy_q5_1_f32", cpy_q5_1_f32_len, cpy_q5_1_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q5_1), 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q8_0], "cpy_q8_0_f32", cpy_q8_0_f32_len, cpy_q8_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q8_0), 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_IQ4_NL], "cpy_iq4_nl_f32", cpy_iq4_nl_f32_len, cpy_iq4_nl_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_IQ4_NL), 1, 1}, {}, 1); + VkPhysicalDeviceFeatures2 device_features2; + device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + device_features2.pNext = nullptr; + device_features2.features = (VkPhysicalDeviceFeatures)device_features; - auto get_suffix = [](bool src0_f16, bool src1_f16, bool dst_f16) { - std::string s; - s += std::string(src0_f16 ? "_f16" : "_f32"); - s += std::string(src1_f16 ? "_f16" : "_f32"); - s += std::string(dst_f16 ? "_f16" : "_f32"); - return s; - }; + VkPhysicalDeviceVulkan11Features vk11_features; + vk11_features.pNext = nullptr; + vk11_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES; + device_features2.pNext = &vk11_features; -#define CREATE_BINARY(name, namemod, spec, bindings) \ - for (int s0 : {0,1}) for (int s1 : {0,1}) for (int d : {0,1}) \ - ggml_vk_create_pipeline2(device, device->pipeline_ ## name ## namemod[s0][s1][d], \ - #name + get_suffix(s0, s1, d) + #namemod, name ## _len[s0][s1][d], name ## _data[s0][s1][d], \ - "main", (bindings), sizeof(vk_op_binary_push_constants), {512, 1, 1}, spec, 1); + VkPhysicalDeviceVulkan12Features vk12_features; + vk12_features.pNext = nullptr; + vk12_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; + vk11_features.pNext = &vk12_features; - CREATE_BINARY(add, , {0}, 4) - CREATE_BINARY(add, _norepeat, {1}, 4) - CREATE_BINARY(sub, , {0}, 3) - CREATE_BINARY(sub, _norepeat, {1}, 3) - CREATE_BINARY(mul, , {0}, 3) - CREATE_BINARY(mul, _norepeat, {1}, 3) - CREATE_BINARY(div, , {0}, 3) - CREATE_BINARY(div, _norepeat, {1}, 3) - CREATE_BINARY(add_rms, , {0}, 4) - CREATE_BINARY(add_rms, _norepeat, {1}, 4) -#undef CREATE_BINARY + last_struct = (VkBaseOutStructure *)&vk12_features; - // f32 multi_add pipelines. f16 variants stay behind rte_fp16 (none today). - if (device->multi_add) { - for (uint32_t i = 0; i < MAX_FUSED_ADDS; ++i) { - ggml_vk_create_pipeline2(device, device->pipeline_multi_add[i], "multi_add_f32_" + std::to_string(i+1), multi_add_f32_len, multi_add_f32_data, "main", MAX_PARAMETER_COUNT, sizeof(vk_op_multi_add_push_constants), {512, 1, 1}, {i+2}, 1); - ggml_vk_create_pipeline2(device, device->pipeline_multi_add_rms[i], "multi_add_rms_f32_" + std::to_string(i+1), multi_add_rms_f32_len, multi_add_rms_f32_data, "main", MAX_PARAMETER_COUNT, sizeof(vk_op_multi_add_push_constants), {512, 1, 1}, {i+2}, 1); + VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR internally_synchronized_queues_features{}; + internally_synchronized_queues_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INTERNALLY_SYNCHRONIZED_QUEUES_FEATURES_KHR; + internally_synchronized_queues_features.pNext = nullptr; + internally_synchronized_queues_features.internallySynchronizedQueues = VK_FALSE; + + if (internally_sync_support) { + last_struct->pNext = (VkBaseOutStructure *)&internally_synchronized_queues_features; + last_struct = (VkBaseOutStructure *)&internally_synchronized_queues_features; + device_extensions.push_back(VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME); } - } - ggml_vk_create_pipeline(device, device->pipeline_add_id_f32, "add_id_f32", add_id_f32_len, add_id_f32_data, "main", 4, sizeof(vk_op_add_id_push_constants), {1, 1, 1}, {}, 1); + VkPhysicalDevicePipelineRobustnessFeaturesEXT pl_robustness_features; + pl_robustness_features.pNext = nullptr; + pl_robustness_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_ROBUSTNESS_FEATURES_EXT; + pl_robustness_features.pipelineRobustness = VK_FALSE; - ggml_vk_create_pipeline(device, device->pipeline_acc_f32, "acc_f32", acc_f32_len, acc_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {0, 1}, 1); - ggml_vk_create_pipeline(device, device->pipeline_set_f32, "set_f32", acc_f32_len, acc_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {0, 0}, 1); + if (pipeline_robustness) { + last_struct->pNext = (VkBaseOutStructure *)&pl_robustness_features; + last_struct = (VkBaseOutStructure *)&pl_robustness_features; + device_extensions.push_back("VK_EXT_pipeline_robustness"); + } - ggml_vk_create_pipeline(device, device->pipeline_concat_i8, "concat_i8", concat_i8_len, concat_i8_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_concat_i16, "concat_i16", concat_i16_len, concat_i16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_concat_i32, "concat_i32", concat_i32_len, concat_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_concat_i64, "concat_i64", concat_i64_len, concat_i64_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); + VkPhysicalDeviceMemoryPriorityFeaturesEXT memory_priority_features; + memory_priority_features.pNext = nullptr; + memory_priority_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PRIORITY_FEATURES_EXT; + memory_priority_features.memoryPriority = VK_FALSE; + if (device->memory_priority) { + last_struct->pNext = (VkBaseOutStructure *)&memory_priority_features; + last_struct = (VkBaseOutStructure *)&memory_priority_features; + device_extensions.push_back("VK_EXT_memory_priority"); + } - ggml_vk_create_pipeline(device, device->pipeline_upscale_nearest_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_NEAREST}, 1); - ggml_vk_create_pipeline(device, device->pipeline_upscale_bilinear_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BILINEAR}, 1); - ggml_vk_create_pipeline(device, device->pipeline_upscale_bicubic_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BICUBIC}, 1); - ggml_vk_create_pipeline(device, device->pipeline_upscale_bilinear_antialias_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS}, 1); + VkPhysicalDeviceSubgroupSizeControlFeaturesEXT subgroup_size_control_features; + subgroup_size_control_features.pNext = nullptr; + subgroup_size_control_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT; + subgroup_size_control_features.computeFullSubgroups = false; + subgroup_size_control_features.subgroupSizeControl = false; - ggml_vk_create_pipeline(device, device->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + if (device->subgroup_size_control) { + last_struct->pNext = (VkBaseOutStructure *)&subgroup_size_control_features; + last_struct = (VkBaseOutStructure *)&subgroup_size_control_features; + } - ggml_vk_create_pipeline(device, device->pipeline_log[0], "log_f32", log_f32_len, log_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_log[1], "log_f16", log_f16_len, log_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); +#if defined(VK_KHR_cooperative_matrix) + VkPhysicalDeviceCooperativeMatrixFeaturesKHR coopmat_features; + coopmat_features.pNext = nullptr; + coopmat_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR; + coopmat_features.cooperativeMatrix = VK_FALSE; - ggml_vk_create_pipeline(device, device->pipeline_tri[0], "tri_f32", tri_f32_len, tri_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_tri[1], "tri_f16", tri_f16_len, tri_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + if (device->coopmat_support) { + last_struct->pNext = (VkBaseOutStructure *)&coopmat_features; + last_struct = (VkBaseOutStructure *)&coopmat_features; + } +#endif - ggml_vk_create_pipeline(device, device->pipeline_diag[0], "diag_f32", diag_f32_len, diag_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_diag[1], "diag_f16", diag_f16_len, diag_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); +#if defined(VK_NV_cooperative_matrix2) + VkPhysicalDeviceCooperativeMatrix2FeaturesNV coopmat2_features {}; + coopmat2_features.pNext = nullptr; + coopmat2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_2_FEATURES_NV; + if (coopmat2_support) { + last_struct->pNext = (VkBaseOutStructure *)&coopmat2_features; + last_struct = (VkBaseOutStructure *)&coopmat2_features; + device_extensions.push_back("VK_NV_cooperative_matrix2"); + } +#endif - ggml_vk_create_pipeline(device, device->pipeline_pad_f32, "pad_f32", pad_f32_len, pad_f32_data, "main", 2, sizeof(vk_op_pad_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_pad_reflect_1d_f32, "pad_reflect_1d_f32", pad_reflect_1d_f32_len, pad_reflect_1d_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV coopmat2_decode_vector_features {}; + coopmat2_decode_vector_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV; + if (coopmat2_decode_vector_support) { + last_struct->pNext = (VkBaseOutStructure *)&coopmat2_decode_vector_features; + last_struct = (VkBaseOutStructure *)&coopmat2_decode_vector_features; + device_extensions.push_back(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME); + } - ggml_vk_create_pipeline(device, device->pipeline_roll_f32, "roll_f32", roll_f32_len, roll_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); +#if defined(VK_KHR_shader_bfloat16) + VkPhysicalDeviceShaderBfloat16FeaturesKHR bfloat16_features {}; + bfloat16_features.pNext = nullptr; + bfloat16_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR; + if (bfloat16_support) { + last_struct->pNext = (VkBaseOutStructure *)&bfloat16_features; + last_struct = (VkBaseOutStructure *)&bfloat16_features; + device_extensions.push_back("VK_KHR_shader_bfloat16"); + } +#endif - ggml_vk_create_pipeline(device, device->pipeline_repeat_i32, "repeat_i32", repeat_i32_len, repeat_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_repeat_back_f32, "repeat_back_f32", repeat_back_f32_len, repeat_back_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT ocp_microscaling_features {}; + ocp_microscaling_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OCP_MICROSCALING_TYPES_FEATURES_EXT; + if (ocp_microscaling_extension) { + last_struct->pNext = (VkBaseOutStructure *)&ocp_microscaling_features; + last_struct = (VkBaseOutStructure *)&ocp_microscaling_features; + device_extensions.push_back(VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME); + } - ggml_vk_create_pipeline(device, device->pipeline_repeat_i16, "repeat_i16", repeat_i16_len, repeat_i16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + VkPhysicalDeviceShaderFloat8FeaturesEXT shader_float8_features {}; + shader_float8_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT; + if (shader_float8_extension) { + last_struct->pNext = (VkBaseOutStructure *)&shader_float8_features; + last_struct = (VkBaseOutStructure *)&shader_float8_features; + device_extensions.push_back(VK_EXT_SHADER_FLOAT8_EXTENSION_NAME); + } -#define CREATE_UNARY(name) \ - ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \ - ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + VkPhysicalDeviceMaintenance4Features maint4_features {}; + maint4_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_4_FEATURES; + if (maintenance4_support) { + last_struct->pNext = (VkBaseOutStructure *)&maint4_features; + last_struct = (VkBaseOutStructure *)&maint4_features; + device_extensions.push_back("VK_KHR_maintenance4"); + } - CREATE_UNARY(elu) - CREATE_UNARY(gelu) - CREATE_UNARY(gelu_erf) - CREATE_UNARY(gelu_quick) - CREATE_UNARY(silu) - CREATE_UNARY(relu) - CREATE_UNARY(sqr) - CREATE_UNARY(sqrt) - CREATE_UNARY(sin) - CREATE_UNARY(cos) - CREATE_UNARY(clamp) - CREATE_UNARY(leaky_relu) - CREATE_UNARY(xielu) - CREATE_UNARY(neg) - CREATE_UNARY(tanh) - CREATE_UNARY(sigmoid) - CREATE_UNARY(hardsigmoid) - CREATE_UNARY(hardswish) - CREATE_UNARY(abs) - CREATE_UNARY(softplus) - CREATE_UNARY(step) - CREATE_UNARY(round) - CREATE_UNARY(ceil) - CREATE_UNARY(floor) - CREATE_UNARY(trunc) - CREATE_UNARY(sgn) - CREATE_UNARY(exp) - CREATE_UNARY(expm1) -#undef CREATE_UNARY + VkPhysicalDeviceShaderIntegerDotProductFeaturesKHR shader_integer_dot_product_features {}; + shader_integer_dot_product_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_INTEGER_DOT_PRODUCT_FEATURES_KHR; + if (device->integer_dot_product) { + last_struct->pNext = (VkBaseOutStructure *)&shader_integer_dot_product_features; + last_struct = (VkBaseOutStructure *)&shader_integer_dot_product_features; + device_extensions.push_back("VK_KHR_shader_integer_dot_product"); + } - ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f16, "add1_f16_f16", add1_f16_f16_len, add1_f16_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f32, "add1_f16_f32", add1_f16_f32_len, add1_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_add1_f32_f32, "add1_f32_f32", add1_f32_f32_len, add1_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); + VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE dot2_features {}; + dot2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE; + if (dot2_f16_support) { + last_struct->pNext = (VkBaseOutStructure *)&dot2_features; + last_struct = (VkBaseOutStructure *)&dot2_features; + device_extensions.push_back("VK_VALVE_shader_mixed_float_dot_product"); + } - ggml_vk_create_pipeline(device, device->pipeline_arange_f32, "arange_f32", arange_f32_len, arange_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + VkPhysicalDevicePipelineExecutablePropertiesFeaturesKHR pep_features {}; + pep_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_EXECUTABLE_PROPERTIES_FEATURES_KHR; + if (pipeline_executable_properties_support) { + last_struct->pNext = (VkBaseOutStructure *)&pep_features; + last_struct = (VkBaseOutStructure *)&pep_features; + device_extensions.push_back("VK_KHR_pipeline_executable_properties"); + } - ggml_vk_create_pipeline(device, device->pipeline_fill_f32, "fill_f32", fill_f32_len, fill_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_fill_f16, "fill_f16", fill_f16_len, fill_f16_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + if (device->external_memory_host) { + device_extensions.push_back("VK_EXT_external_memory_host"); + } -#define CREATE_GLU(name) \ - ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); + if (device->external_semaphore_fd) { + device_extensions.push_back("VK_KHR_external_semaphore_fd"); + } - CREATE_GLU(geglu) - CREATE_GLU(reglu) - CREATE_GLU(swiglu) - CREATE_GLU(swiglu_oai) - CREATE_GLU(swiglu_clamp) - CREATE_GLU(geglu_erf) - CREATE_GLU(geglu_quick) -#undef CREATE_GLU +#if defined(VK_EXT_shader_64bit_indexing) + VkPhysicalDeviceShader64BitIndexingFeaturesEXT shader_64bit_indexing_features {}; + shader_64bit_indexing_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_64_BIT_INDEXING_FEATURES_EXT; + if (device->shader_64b_indexing) { + last_struct->pNext = (VkBaseOutStructure *)&shader_64bit_indexing_features; + last_struct = (VkBaseOutStructure *)&shader_64bit_indexing_features; + device_extensions.push_back("VK_EXT_shader_64bit_indexing"); + } +#endif - ggml_vk_create_pipeline(device, device->pipeline_silu_back_f32, "silu_back_f32", silu_back_f32_len, silu_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + VkPhysicalDeviceFaultFeaturesEXT fault_features {}; + fault_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FAULT_FEATURES_EXT; + if (device->device_fault) { + last_struct->pNext = (VkBaseOutStructure *)&fault_features; + last_struct = (VkBaseOutStructure *)&fault_features; + device_extensions.push_back("VK_EXT_device_fault"); + } - ggml_vk_create_pipeline(device, device->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {1, 512, 1}, {}, 1, true); + vkGetPhysicalDeviceFeatures2(device->physical_device, &device_features2); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32, "soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_wg512, "soft_max_f32_wg512", soft_max_f32_len, soft_max_f32_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 512 }, 1); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_f16, "soft_max_f32_f16", soft_max_f32_f16_len, soft_max_f32_f16_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_f16_wg512, "soft_max_f32_f16_wg512", soft_max_f32_f16_len, soft_max_f32_f16_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 512 }, 1); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_back_f32, "soft_max_back_f32", soft_max_back_f32_len, soft_max_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1, true); + device->device_fault = device->device_fault && fault_features.deviceFault; - ggml_vk_create_pipeline(device, device->pipeline_soft_max_large1_f32, "soft_max_large1_f32", soft_max_large1_f32_len, soft_max_large1_f32_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_large2_f32, "soft_max_large2_f32", soft_max_large2_f32_len, soft_max_large2_f32_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_large3_f32, "soft_max_large3_f32", soft_max_large3_f32_len, soft_max_large3_f32_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_large1_f32_f16, "soft_max_large1_f32_f16", soft_max_large1_f32_f16_len, soft_max_large1_f32_f16_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_large2_f32_f16, "soft_max_large2_f32_f16", soft_max_large2_f32_f16_len, soft_max_large2_f32_f16_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_large3_f32_f16, "soft_max_large3_f32_f16", soft_max_large3_f32_f16_len, soft_max_large3_f32_f16_data, "main", 6, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 128, 4 }, 1, true); + device->has_internally_synchronized_queues = internally_synchronized_queues_features.internallySynchronizedQueues; - ggml_vk_create_pipeline(device, device->pipeline_rope_norm_f32, "rope_norm_f32", rope_norm_f32_len, rope_norm_f32_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_rope_neox_f32, "rope_neox_f32", rope_neox_f32_len, rope_neox_f32_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_rope_multi_f32, "rope_multi_f32", rope_multi_f32_len, rope_multi_f32_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_rope_vision_f32, "rope_vision_f32", rope_vision_f32_len, rope_vision_f32_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + // Build queue create infos only after querying whether internally synchronized queues are enabled. + // getQueue2() later uses the same flag, so creation/retrieval must stay consistent. + vk::DeviceQueueCreateFlags queue_flags = device->has_internally_synchronized_queues ? + eInternallySynchronizedKHR : + vk::DeviceQueueCreateFlags(); - ggml_vk_create_pipeline(device, device->pipeline_rope_norm_f16, "rope_norm_f16", rope_norm_f16_len, rope_norm_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_rope_neox_f16, "rope_neox_f16", rope_neox_f16_len, rope_neox_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_rope_multi_f16, "rope_multi_f16", rope_multi_f16_len, rope_multi_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_rope_vision_f16, "rope_vision_f16", rope_vision_f16_len, rope_vision_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + if (compute_queue_family_index != transfer_queue_family_index) { + device_queue_create_infos.push_back({queue_flags, compute_queue_family_index, 1, priorities}); + device_queue_create_infos.push_back({queue_flags, transfer_queue_family_index, 1, priorities + 1}); + } else if(!device->single_queue) { + device_queue_create_infos.push_back({queue_flags, compute_queue_family_index, 2, priorities}); + } else { + device_queue_create_infos.push_back({queue_flags, compute_queue_family_index, 1, priorities}); + } - ggml_vk_create_pipeline(device, device->pipeline_rope_norm_f32_f16, "rope_norm_f32_f16", rope_norm_f32_f16_len, rope_norm_f32_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_rope_neox_f32_f16, "rope_neox_f32_f16", rope_neox_f32_f16_len, rope_neox_f32_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_rope_multi_f32_f16, "rope_multi_f32_f16", rope_multi_f32_f16_len, rope_multi_f32_f16_data, "main", 5, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + device->pipeline_executable_properties_support = pipeline_executable_properties_support; - for (uint32_t i = 0; i < num_argsort_pipelines; ++i) { - uint32_t BLOCK_SIZE = 1u << std::min(i, device->max_workgroup_size_log2); - if (i <= device->max_workgroup_size_log2 && - 2 * sizeof(int) * BLOCK_SIZE <= device->properties.limits.maxComputeSharedMemorySize) { - const uint32_t NCOLS_PADDED_LOG2 = i; - ggml_vk_create_pipeline2(device, device->pipeline_argsort_f32[i], "argsort_f32_"+std::to_string(i), argsort_f32_len, argsort_f32_data, "main", 3, sizeof(vk_op_argsort_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, NCOLS_PADDED_LOG2}, 1, true); - } - const uint32_t WG_UNROLL_FACTOR = BLOCK_SIZE > 1 ? 2 : 1; - BLOCK_SIZE /= WG_UNROLL_FACTOR; - ggml_vk_create_pipeline2(device, device->pipeline_argsort_large_f32[i], "argsort_large_f32_"+std::to_string(i), argsort_large_f32_len, argsort_large_f32_data, "main", 3, sizeof(vk_op_argsort_push_constants), {BLOCK_SIZE * WG_UNROLL_FACTOR, 1, 1}, {BLOCK_SIZE, WG_UNROLL_FACTOR}, 1, true); - } + device->fp16 = device->fp16 && vk12_features.shaderFloat16; - for (uint32_t i = 0; i < num_topk_pipelines; ++i) { - const uint32_t BLOCK_SIZE = 1u << i; - const uint32_t NCOLS_PADDED_LOG2 = i; - if (i <= device->max_workgroup_size_log2) { - uint32_t nary_shmem = 2 * sizeof(int) * BLOCK_SIZE + - sizeof(int) * device->subgroup_size + - 2 * sizeof(int) + - 2 * (BLOCK_SIZE / device->subgroup_size) * sizeof(int); - if (device->subgroup_arithmetic && device->subgroup_require_full_support && device->subgroup_shuffle && device->subgroup_ballot && - nary_shmem <= device->properties.limits.maxComputeSharedMemorySize) { - ggml_vk_create_pipeline2(device, device->pipeline_topk_f32[i], "topk_f32_"+std::to_string(i), topk_nary_search_f32_len, topk_nary_search_f32_data, "main", 2, sizeof(vk_op_topk_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, device->subgroup_size, device->subgroup_size_log2}, 1, true, true, device->subgroup_size); - } else if (2 * sizeof(int) * BLOCK_SIZE <= device->properties.limits.maxComputeSharedMemorySize) { - ggml_vk_create_pipeline2(device, device->pipeline_topk_f32[i], "topk_f32_"+std::to_string(i), topk_argsort_f32_len, topk_argsort_f32_data, "main", 2, sizeof(vk_op_topk_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, NCOLS_PADDED_LOG2}, 1, true); - } - } - } +#if defined(VK_KHR_shader_bfloat16) + device->bf16 = bfloat16_support && bfloat16_features.shaderBFloat16Type; +#else + device->bf16 = false; +#endif - // large-k fallback: one workgroup per row, radix-select instead of a full sort. The QSA - // variant (spec constant 1) additionally gathers the qwen4 indexer input on the fly. - { - const uint32_t BLOCK_SIZE = 1u << std::min(10u, device->max_workgroup_size_log2); - ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_f32, "topk_radix_f32", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 0}, 1, true); - ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_qsa, "topk_radix_qsa", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 1}, 1, true); - } + device->dot2_f16 = dot2_f16_support && dot2_features.shaderMixedFloatDotProductFloat16AccFloat32; + device->ocp_fp4 = ocp_microscaling_extension && ocp_microscaling_features.shaderFloat4 && + shader_float8_extension && shader_float8_features.shaderFloat8 && + !getenv("GGML_VK_DISABLE_OCP_FP4"); - ggml_vk_create_pipeline(device, device->pipeline_argmax_f32, "argmax_f32", argmax_f32_len, argmax_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); + device->pipeline_robustness = pl_robustness_features.pipelineRobustness; - ggml_vk_create_pipeline(device, device->pipeline_sum_rows_f32, "sum_rows_f32", sum_rows_f32_len, sum_rows_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); - ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_f32, "cross_entropy_loss_f32", cross_entropy_loss_f32_len, cross_entropy_loss_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); - ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_f32_wg512, "cross_entropy_loss_f32_wg512", cross_entropy_loss_f32_len, cross_entropy_loss_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { 512 }, 1); - ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_back_f32, "cross_entropy_loss_back_f32", cross_entropy_loss_back_f32_len, cross_entropy_loss_back_f32_data, "main", 4, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); - ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_back_f32_wg512, "cross_entropy_loss_back_f32_wg512", cross_entropy_loss_back_f32_len, cross_entropy_loss_back_f32_data, "main", 4, sizeof(vk_op_push_constants), {1, 1, 1}, { 512 }, 1); - // Intel Windows driver in range [32.0.101.8509, 32.0.101.8860) will crash when using fwht kernels so we gate that here - const bool can_use_fwht = device->driver_id != vk::DriverId::eIntelProprietaryWindows || - !ggml_vk_intel_windows_driver_in_range(device->properties.driverVersion, 101, 8509, 101, 8860); - if (can_use_fwht && device->subgroup_basic && device->subgroup_shuffle) { - int idx = 0; - for (uint32_t n : {64, 128, 256, 512}) { - if (device->subgroup_size <= n) { - ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_f32", fwht_f32_len, fwht_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { device->subgroup_size, n }, 1, true, true, device->subgroup_size); - } - ++idx; - } - } else if (can_use_fwht) { - int idx = 0; - for (uint32_t n : {64, 128, 256, 512}) { - const uint32_t block_size = std::min(device->subgroup_size, n); - ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_shmem_f32", fwht_shmem_f32_len, fwht_shmem_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { block_size, n }, 1); - ++idx; + // f32 multi_add needs 16-bit storage, not fp16 RTE. Default = RTE gate. + // GGML_VK_MULTI_ADD_F32_NO_RTE=1 creates f32 pipelines without RTE. + const bool multi_add_common = + device->properties.limits.maxPushConstantsSize >= sizeof(vk_op_multi_add_push_constants) && + getenv("GGML_VK_DISABLE_MULTI_ADD") == nullptr; + const char * f32_no_rte_env = getenv("GGML_VK_MULTI_ADD_F32_NO_RTE"); + const bool f32_no_rte = + f32_no_rte_env != nullptr && f32_no_rte_env[0] == '1' && + vk11_features.storageBuffer16BitAccess; + device->multi_add = multi_add_common && + (device->float_controls_rte_fp16 || f32_no_rte); + + device->shader_int64 = device_features2.features.shaderInt64; + device->buffer_device_address = vk12_features.bufferDeviceAddress; + device->vulkan_memory_model = vk12_features.vulkanMemoryModel; + + if (device->subgroup_size_control) { + device->subgroup_min_size = subgroup_size_control_props.minSubgroupSize; + device->subgroup_max_size = subgroup_size_control_props.maxSubgroupSize; + device_extensions.push_back("VK_EXT_subgroup_size_control"); } - } - const uint32_t cumsum_elem_per_thread = (device->vendor_id == VK_VENDOR_ID_AMD || device->vendor_id == VK_VENDOR_ID_INTEL) ? 2 : 4; - ggml_vk_create_pipeline(device, device->pipeline_cumsum_f32, "cumsum_f32", cumsum_f32_len, cumsum_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { 256, device->subgroup_size, cumsum_elem_per_thread }, 1, true, true, device->subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_cumsum_small_f32, "cumsum_f32", cumsum_f32_len, cumsum_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { 128, device->subgroup_size, 1 }, 1, true, true, device->subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_cumsum_multipass1_f32, "cumsum_multipass1_f32", cumsum_multipass1_f32_len, cumsum_multipass1_f32_data, "main", 3, sizeof(vk_op_sum_rows_push_constants), {256, 1, 1}, { 256, device->subgroup_size }, 1, true, true, device->subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_cumsum_multipass2_f32, "cumsum_multipass2_f32", cumsum_multipass2_f32_len, cumsum_multipass2_f32_data, "main", 3, sizeof(vk_op_sum_rows_push_constants), {256, 1, 1}, { 256, device->subgroup_size }, 1, true, true, device->subgroup_size); + device->subgroup_size_control = device->subgroup_size_control && + (subgroup_size_control_props.requiredSubgroupSizeStages & vk::ShaderStageFlagBits::eCompute) && + subgroup_size_control_features.subgroupSizeControl; - ggml_vk_create_pipeline(device, device->pipeline_count_equal_i32, "count_equal_i32", count_equal_i32_len, count_equal_i32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, { device->subgroup_size }, 1); + device->subgroup_require_full_support = subgroup_size_control_features.computeFullSubgroups; - if (device->subgroup_arithmetic && device->subgroup_require_full_support) { - ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_subgroup_len, count_experts_subgroup_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true, true); - } else { - ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_len, count_experts_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true); - } +#if defined(VK_KHR_cooperative_matrix) + device->coopmat_support = device->coopmat_support && coopmat_features.cooperativeMatrix; + device->coopmat1_fa_support = device->coopmat_support && device->subgroup_require_full_support; +#endif - for (auto &s : device->pipeline_solve_tri_f32) { - const vk_solve_tri_pipeline_state &state = s.first; + if (coopmat2_support) { +#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) + if (coopmat2_features.cooperativeMatrixWorkgroupScope && + coopmat2_features.cooperativeMatrixFlexibleDimensions && + coopmat2_features.cooperativeMatrixReductions && + coopmat2_features.cooperativeMatrixConversions && + coopmat2_features.cooperativeMatrixPerElementOperations && + coopmat2_features.cooperativeMatrixTensorAddressing && + coopmat2_features.cooperativeMatrixBlockLoads && + vk12_features.bufferDeviceAddress) { - // Max number of rows to load at a time, limited by shared memory - const uint32_t batch_N = device->properties.limits.maxComputeSharedMemorySize / ((state.N + state.K) * sizeof(float)); - // Need at least K invocations, and prefer a minimum of 128 to spread out loading shared memory - const uint32_t block_size = std::max(128u, 1u << (uint32_t)ceilf(log2f(float(state.K)))); + std::vector<VkCooperativeMatrixFlexibleDimensionsPropertiesNV> flexible_dimensions; + uint32_t count = 0; - ggml_vk_create_pipeline( - device, s.second, "solve_tri_f32", - solve_tri_f32_len, solve_tri_f32_data, "main", 3, - sizeof(vk_op_binary_push_constants), {1, 1, 1}, { 0, state.N, state.K, batch_N, block_size }, 1, true); - } + PFN_vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV + _vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV = + (PFN_vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV) + vk_instance.instance.getProcAddr("vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV"); -#define IM2COL(bda) \ - ggml_vk_create_pipeline(device, device->pipeline_im2col_f32, "im2col_f32", im2col_f32 ## bda ## _len, im2col_f32 ## bda ## _data, "main", 2, sizeof(vk_op_im2col_push_constants), {512, 1, 1}, { device->subgroup_size }, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_im2col_3d_f32, "im2col_3d_f32", im2col_3d_f32 ## bda ## _len, im2col_3d_f32 ## bda ## _data, "main", 2, sizeof(vk_op_im2col_3d_push_constants), {512, 1, 1}, { 512 }, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_im2col_f32_f16, "im2col_f32_f16", im2col_f32_f16 ## bda ## _len, im2col_f32_f16 ## bda ## _data, "main", 2, sizeof(vk_op_im2col_push_constants), {512, 1, 1}, { device->subgroup_size }, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_im2col_3d_f32_f16, "im2col_3d_f32_f16", im2col_3d_f32_f16 ## bda ## _len, im2col_3d_f32_f16 ## bda ## _data, "main", 2, sizeof(vk_op_im2col_3d_push_constants), {512, 1, 1}, { 512 }, 1, true); - if (device->shader_int64 && device->buffer_device_address) { - IM2COL(_bda) - } else { - IM2COL() - } + _vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV(device->physical_device, &count, nullptr); - ggml_vk_create_pipeline(device, device->pipeline_timestep_embedding_f32, "timestep_embedding_f32", timestep_embedding_f32_len, timestep_embedding_f32_data, "main", 2, sizeof(vk_op_timestep_embedding_push_constants), {256, 1, 1}, {}, 1); + VkCooperativeMatrixFlexibleDimensionsPropertiesNV empty_prop {}; + empty_prop.sType = VK_STRUCTURE_TYPE_COOPERATIVE_MATRIX_FLEXIBLE_DIMENSIONS_PROPERTIES_NV; + flexible_dimensions.resize(count, empty_prop); - ggml_vk_create_pipeline(device, device->pipeline_conv_transpose_1d_f32, "conv_transpose_1d_f32", conv_transpose_1d_f32_len, conv_transpose_1d_f32_data, "main", 3, sizeof(vk_op_conv_transpose_1d_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_col2im_1d_f32, "col2im_1d_f32", col2im_1d_f32_len, col2im_1d_f32_data, "main", 2, sizeof(vk_op_col2im_1d_push_constants), {256, 1, 1}, {}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_col2im_1d_f16, "col2im_1d_f16", col2im_1d_f16_len, col2im_1d_f16_data, "main", 2, sizeof(vk_op_col2im_1d_push_constants), {256, 1, 1}, {}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_col2im_1d_bf16, "col2im_1d_bf16", col2im_1d_bf16_len, col2im_1d_bf16_data, "main", 2, sizeof(vk_op_col2im_1d_push_constants), {256, 1, 1}, {}, 1, true); + _vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV(device->physical_device, &count, flexible_dimensions.data()); - ggml_vk_create_pipeline(device, device->pipeline_out_prod_f32, "out_prod_f32", out_prod_f32_len, out_prod_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {256, 1, 1}, {}, 1); + bool found_fp16_128 = false, + found_fp16_256 = false, + found_fp32_128 = false, + found_fp32_256 = false; + bool found_bf16_128 = false, + found_bf16_256 = false; + // need to support fp16*fp16 with fp16/fp32 accumulator, for workgroupsize 128 + // with 32x16x16 and 256 with 32x32x16. + for (auto &prop : flexible_dimensions) { + if (prop.saturatingAccumulation == VK_FALSE && + prop.scope == VK_SCOPE_WORKGROUP_KHR) { - ggml_vk_create_pipeline(device, device->pipeline_snake_f32, "snake_f32", snake_f32_len, snake_f32_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_snake_f16, "snake_f16", snake_f16_len, snake_f16_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_snake_bf16, "snake_bf16", snake_bf16_len, snake_bf16_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1); + if (prop.AType == VK_COMPONENT_TYPE_FLOAT16_KHR && + prop.BType == VK_COMPONENT_TYPE_FLOAT16_KHR) { - ggml_vk_create_pipeline(device, device->pipeline_pool1d_f32, "pool1d_f32", pool1d_f32_len, pool1d_f32_data, "main", 2, sizeof(vk_op_pool1d_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_pool2d_f32, "pool2d_f32", pool2d_f32_len, pool2d_f32_data, "main", 2, sizeof(vk_op_pool2d_push_constants), {512, 1, 1}, {}, 1); + if (prop.workgroupInvocations == 128 && + prop.MGranularity <= 32 && + prop.NGranularity <= 16 && + prop.KGranularity <= 16) { + if (prop.CType == VK_COMPONENT_TYPE_FLOAT16_KHR && + prop.ResultType == VK_COMPONENT_TYPE_FLOAT16_KHR) { + found_fp16_128 = true; + } + if (prop.CType == VK_COMPONENT_TYPE_FLOAT32_KHR && + prop.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR) { + found_fp32_128 = true; + } + } + if (prop.workgroupInvocations == 256 && + prop.MGranularity <= 32 && + prop.NGranularity <= 32 && + prop.KGranularity <= 16) { + if (prop.CType == VK_COMPONENT_TYPE_FLOAT16_KHR && + prop.ResultType == VK_COMPONENT_TYPE_FLOAT16_KHR) { + found_fp16_256 = true; + } + if (prop.CType == VK_COMPONENT_TYPE_FLOAT32_KHR && + prop.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR) { + found_fp32_256 = true; + } + } + } - ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv6_f32, "rwkv_wkv6_f32", rwkv_wkv6_f32_len, rwkv_wkv6_f32_data, "main", 7, sizeof(vk_op_rwkv_wkv6_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); +#if defined(VK_KHR_shader_bfloat16) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + if (bfloat16_support && + prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR && + prop.BType == VK_COMPONENT_TYPE_BFLOAT16_KHR && + prop.CType == VK_COMPONENT_TYPE_FLOAT32_KHR && + prop.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR) { - ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv7_f32, "rwkv_wkv7_f32", rwkv_wkv7_f32_len, rwkv_wkv7_f32_data, "main", 8, sizeof(vk_op_rwkv_wkv7_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); + if (prop.workgroupInvocations == 128 && + prop.MGranularity <= 32 && + prop.NGranularity <= 16 && + prop.KGranularity <= 16) { + found_bf16_128 = true; + } + if (prop.workgroupInvocations == 256 && + prop.MGranularity <= 32 && + prop.NGranularity <= 32 && + prop.KGranularity <= 16) { + found_bf16_256 = true; + } + } +#endif + } + } + if (found_fp16_128 && found_fp16_256 && + found_fp32_128 && found_fp32_256 && + coopmat2_props.cooperativeMatrixFlexibleDimensionsMaxDimension >= 512) { + device->coopmat2 = true; + device->coopmat2_bf16_support = found_bf16_128 && found_bf16_256; + device->coopmat2_decode_vector = coopmat2_decode_vector_support && coopmat2_decode_vector_features.cooperativeMatrixDecodeVector; + } + } +#endif + } - ggml_vk_create_pipeline(device, device->pipeline_gated_linear_attn_f32, "gated_linear_attn_f32", gated_linear_attn_f32_len, gated_linear_attn_f32_data, "main", 6, sizeof(vk_op_gated_linear_attn_push_constants), {1, 1, 1}, {}, 1); + if (!vk11_features.storageBuffer16BitAccess) { + std::cerr << "ggml_vulkan: device " << GGML_VK_NAME << idx << " does not support 16-bit storage." << std::endl; + throw std::runtime_error("Unsupported device"); + } - { - const bool li_subgroup = device->subgroup_arithmetic && device->subgroup_require_full_support; - const size_t li_len = li_subgroup ? lightning_indexer_subgroup_f32_len : lightning_indexer_f32_len; - const void * li_data = li_subgroup ? (const void *)lightning_indexer_subgroup_f32_data : (const void *)lightning_indexer_f32_data; + device_extensions.push_back("VK_KHR_16bit_storage"); - for (ggml_type k_type : lightning_indexer_k_types) { - const std::string name = "lightning_indexer_" + std::string(ggml_type_name(k_type)) + "_k_f32"; - ggml_vk_create_pipeline(device, device->pipeline_lightning_indexer_f32[k_type], name.c_str(), li_len, li_data, "main", 5, sizeof(vk_op_lightning_indexer_push_constants), {1, 1, 1}, {(uint32_t)k_type, fa_block_bytes(k_type), device->subgroup_size}, 1, true, li_subgroup); +#ifdef GGML_VULKAN_VALIDATE + device_extensions.push_back("VK_KHR_shader_non_semantic_info"); +#endif + + if (device->fp16) { + device_extensions.push_back("VK_KHR_shader_float16_int8"); } - } - { - const uint32_t gdn_sizes[] = {16, 32, 64, 128}; - const char * gdn_names[][2] = { - {"gated_delta_net_f32_d16", "gated_delta_net_f32_d16_kda"}, - {"gated_delta_net_f32_d32", "gated_delta_net_f32_d32_kda"}, - {"gated_delta_net_f32_d64", "gated_delta_net_f32_d64_kda"}, - {"gated_delta_net_f32_d128", "gated_delta_net_f32_d128_kda"}, - }; - const char * gdn_chunked_names[] = { - "gated_delta_net_chunked_f32_d16", - "gated_delta_net_chunked_f32_d32", - "gated_delta_net_chunked_f32_d64", - "gated_delta_net_chunked_f32_d128", - }; - for (uint32_t si = 0; si < 4; si++) { - const uint32_t S_V = gdn_sizes[si]; - GGML_ASSERT(is_pow2(S_V)); +#if defined(VK_KHR_cooperative_matrix) + if (device->coopmat_support) { + // Query supported shapes + std::vector<VkCooperativeMatrixPropertiesKHR> cm_props; - uint32_t lanes_per_column; - if (S_V >= 128u && device->subgroup_clustered) { - lanes_per_column = 8u; - } else { - // Use largest power-of-two that divides both S_V and subgroup_size so that - // (1) S_V % lanes_per_column == 0 and (2) S_V % (subgroup_size / lanes_per_column) == 0. - // This means we don't need extra bounds checking logic in the shader. - lanes_per_column = std::min(S_V, device->subgroup_size); - } + PFN_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR pfn_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR = + (PFN_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR)vkGetInstanceProcAddr(vk_instance.instance, "vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR"); - // gated_delta_net.comp relies on S_V % COLS_PER_WG == 0 and - // S_V % LANES_PER_COLUMN == 0 to avoid bounds checks. - while (lanes_per_column > 1u) { - const bool valid_lanes = (device->subgroup_size % lanes_per_column) == 0 && - (S_V % lanes_per_column) == 0; - const uint32_t cols_per_wg = valid_lanes ? device->subgroup_size / lanes_per_column : 0; - if (valid_lanes && cols_per_wg > 0 && (S_V % cols_per_wg) == 0) { - break; - } - lanes_per_column >>= 1u; + uint32_t cm_props_num; + + pfn_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR(device->physical_device, &cm_props_num, nullptr); + + cm_props.resize(cm_props_num); + + for (auto& prop : cm_props) { + prop.sType = VK_STRUCTURE_TYPE_COOPERATIVE_MATRIX_PROPERTIES_KHR; } - GGML_ASSERT((device->subgroup_size % lanes_per_column) == 0); - GGML_ASSERT((S_V % lanes_per_column) == 0); - GGML_ASSERT((S_V % (device->subgroup_size / lanes_per_column)) == 0); + pfn_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR(device->physical_device, &cm_props_num, cm_props.data()); - const bool need_partial_subgroup_reduce = lanes_per_column != 1u && lanes_per_column < device->subgroup_size; - const bool use_clustered_reduce = device->subgroup_arithmetic && device->subgroup_clustered && need_partial_subgroup_reduce; - const bool use_subgroup_reduce = device->subgroup_arithmetic && !need_partial_subgroup_reduce; - // Barrier-free fallback: the shared-memory reduction costs three workgroup - // barriers plus an LDS round trip per butterfly level, which dominates the - // per-token cost of the recurrence on older GPUs. Subgroup shuffle needs - // neither and is available on every GCN/RDNA driver. - const bool use_shuffle_reduce = !use_clustered_reduce && !use_subgroup_reduce && device->subgroup_shuffle; - const bool use_subgroup_ops = use_clustered_reduce || use_subgroup_reduce || use_shuffle_reduce; - size_t gdn_len; - const void * gdn_data; - if (use_clustered_reduce) { - gdn_len = gated_delta_net_f32_len; - gdn_data = (const void *)gated_delta_net_f32_data; - } else if (use_subgroup_reduce) { - gdn_len = gated_delta_net_f32_nocluster_len; - gdn_data = (const void *)gated_delta_net_f32_nocluster_data; - } else if (use_shuffle_reduce) { - gdn_len = gated_delta_net_f32_shuffle_len; - gdn_data = (const void *)gated_delta_net_f32_shuffle_data; - } else { - gdn_len = gated_delta_net_f32_shmem_len; - gdn_data = (const void *)gated_delta_net_f32_shmem_data; - } - - const uint32_t cols_per_wg = device->subgroup_size / lanes_per_column; - const std::array<uint32_t, 3> wg_denoms = {1u, 1u, cols_per_wg}; + VK_LOG_DEBUG("ggml_vulkan: Cooperative Matrix Shapes: " << cm_props.size()); - for (uint32_t kda = 0; kda < 2; kda++) { - ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net[si][kda], - gdn_names[si][kda], gdn_len, gdn_data, "main", 7, sizeof(vk_op_gated_delta_net_push_constants), - wg_denoms, {S_V, kda, device->subgroup_size, lanes_per_column}, 1, true, use_subgroup_ops, device->subgroup_size); - } + for (auto& prop : cm_props) { + VK_LOG_DEBUG("ggml_vulkan: M: " << prop.MSize << " N: " << prop.NSize << " K: " << prop.KSize << " A: " << vk::to_string((vk::ComponentTypeKHR)prop.AType) << " B: " << vk::to_string((vk::ComponentTypeKHR)prop.BType) << " C: " << vk::to_string((vk::ComponentTypeKHR)prop.CType) << " Result: " << vk::to_string((vk::ComponentTypeKHR)prop.ResultType) << " saturatingAccumulation: " << prop.saturatingAccumulation << " scope: " << vk::to_string((vk::ScopeKHR)prop.scope)); - // Chunked (UT-transform) variant for short token blocks. Uses subgroup shuffle - // butterflies only, so it needs neither subgroup arithmetic nor clustered ops. - // The workgroup spans several subgroups to amortise the LDS staging of k/q and - // the Gram blocks over more output columns. - device->gated_delta_net_chunked_cols[si] = 0; - if (device->subgroup_shuffle && device->subgroup_require_full_support) { - uint32_t nsg = 1; - while (nsg < 4 && - cols_per_wg * nsg * 2 <= S_V && - (S_V % (cols_per_wg * nsg * 2)) == 0 && - device->subgroup_size * nsg * 2 <= device->properties.limits.maxComputeWorkGroupInvocations) { - nsg *= 2; + if ((vk::ComponentTypeKHR)prop.AType == vk::ComponentTypeKHR::eFloat16 && + (vk::ComponentTypeKHR)prop.BType == vk::ComponentTypeKHR::eFloat16 && + (vk::ScopeKHR)prop.scope == vk::ScopeKHR::eSubgroup + ) { + if ((vk::ComponentTypeKHR)prop.CType == vk::ComponentTypeKHR::eFloat32 && + (vk::ComponentTypeKHR)prop.ResultType == vk::ComponentTypeKHR::eFloat32) { + // coopmat sizes not set yet + if (device->coopmat_m == 0) { + device->coopmat_acc_f32_support = true; + device->coopmat_m = prop.MSize; + device->coopmat_n = prop.NSize; + device->coopmat_k = prop.KSize; + } else if (device->coopmat_m == prop.MSize && device->coopmat_n == prop.NSize && device->coopmat_k == prop.KSize) { + // Only enable if shape is identical + device->coopmat_acc_f32_support = true; + } + if (prop.MSize == 16 && prop.NSize == 16 && prop.KSize == 16) { + device->coopmat_support_16x16x16_f32acc = true; + } + } else if ((vk::ComponentTypeKHR)prop.CType == vk::ComponentTypeKHR::eFloat16 && + (vk::ComponentTypeKHR)prop.ResultType == vk::ComponentTypeKHR::eFloat16) { + // coopmat sizes not set yet + if (device->coopmat_m == 0) { + device->coopmat_acc_f16_support = true; + device->coopmat_m = prop.MSize; + device->coopmat_n = prop.NSize; + device->coopmat_k = prop.KSize; + } else if (device->coopmat_m == prop.MSize && device->coopmat_n == prop.NSize && device->coopmat_k == prop.KSize) { + // Only enable if shape is identical + device->coopmat_acc_f16_support = true; + } + if (prop.MSize == 16 && prop.NSize == 16 && prop.KSize == 16) { + device->coopmat_support_16x16x16_f16acc = true; + } + } + } else if ((vk::ComponentTypeKHR)prop.AType == vk::ComponentTypeKHR::eSint8 && + (vk::ComponentTypeKHR)prop.BType == vk::ComponentTypeKHR::eSint8 && + (vk::ComponentTypeKHR)prop.CType == vk::ComponentTypeKHR::eSint32 && + (vk::ComponentTypeKHR)prop.ResultType == vk::ComponentTypeKHR::eSint32 && + (vk::ScopeKHR)prop.scope == vk::ScopeKHR::eSubgroup && + device->coopmat_int_m == 0 + ) { + device->coopmat_int_support = true; + device->coopmat_int_m = prop.MSize; + device->coopmat_int_n = prop.NSize; + device->coopmat_int_k = prop.KSize; } - - const uint32_t cols_per_block = cols_per_wg * nsg; - - // 2 staging buffers of T_MAX*S_V floats + the two T_MAX^2 Gram blocks + 3*T_MAX scalars - const size_t chunked_lds = (size_t) 2 * 16 * S_V * sizeof(float) + - (size_t) 2 * 16 * 16 * sizeof(float) + - (size_t) 3 * 16 * sizeof(float); - - if ((S_V % cols_per_block) == 0 && chunked_lds <= device->properties.limits.maxComputeSharedMemorySize) { - const std::array<uint32_t, 3> chunked_wg_denoms = {1u, 1u, cols_per_block}; - ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_chunked[si], - gdn_chunked_names[si], gated_delta_net_chunked_f32_len, gated_delta_net_chunked_f32_data, - "main", 7, sizeof(vk_op_gated_delta_net_push_constants), - chunked_wg_denoms, {S_V, device->subgroup_size, lanes_per_column, nsg}, - 1, true, true, device->subgroup_size); - device->gated_delta_net_chunked_cols[si] = cols_per_block; +#if defined(VK_KHR_shader_bfloat16) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + if (bfloat16_support && + prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR && + prop.BType == VK_COMPONENT_TYPE_BFLOAT16_KHR && + prop.CType == VK_COMPONENT_TYPE_FLOAT32_KHR && + prop.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR && + (vk::ScopeKHR)prop.scope == vk::ScopeKHR::eSubgroup + ) { + // coopmat sizes not set yet + if (device->coopmat_m == 0) { + device->coopmat_bf16_support = true; + device->coopmat_m = prop.MSize; + device->coopmat_n = prop.NSize; + device->coopmat_k = prop.KSize; + } else if (device->coopmat_m == prop.MSize && device->coopmat_n == prop.NSize && device->coopmat_k == prop.KSize) { + // Only enable if shape is identical + device->coopmat_bf16_support = true; + } } +#endif } - } - } - if (device->subgroup_arithmetic && device->subgroup_require_full_support) { - ggml_vk_create_pipeline(device, device->pipeline_ssm_scan_f32_d128, "ssm_scan_128_f32", ssm_scan_subgroup_f32_len, ssm_scan_subgroup_f32_data, "main", 8, sizeof(vk_op_ssm_scan_push_constants), {1, 1, 1}, {128, device->subgroup_size}, 1, true, true); - ggml_vk_create_pipeline(device, device->pipeline_ssm_scan_f32_d256, "ssm_scan_256_f32", ssm_scan_subgroup_f32_len, ssm_scan_subgroup_f32_data, "main", 8, sizeof(vk_op_ssm_scan_push_constants), {1, 1, 1}, {256, device->subgroup_size}, 1, true, true); - } else { - ggml_vk_create_pipeline(device, device->pipeline_ssm_scan_f32_d128, "ssm_scan_128_f32", ssm_scan_f32_len, ssm_scan_f32_data, "main", 8, sizeof(vk_op_ssm_scan_push_constants), {1, 1, 1}, {128, device->subgroup_size, 16}, 1, true, true); - ggml_vk_create_pipeline(device, device->pipeline_ssm_scan_f32_d256, "ssm_scan_256_f32", ssm_scan_f32_len, ssm_scan_f32_data, "main", 8, sizeof(vk_op_ssm_scan_push_constants), {1, 1, 1}, {256, device->subgroup_size, 16}, 1, true, true); - } + if (device->coopmat_m == 0 || !device->coopmat_acc_f32_support) { + // No suitable matmul mode found + GGML_LOG_DEBUG("ggml_vulkan: WARNING: No suitable matrix core mode found. Disabling matrix cores.\n"); + device->coopmat_support = false; + } + } - ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_f32, "ssm_conv_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 4, sizeof(vk_op_ssm_conv_push_constants), {32, 16, 1}, {32, 16, 0, 0}, 1); - ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_silu_f32, "ssm_conv_silu_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 4, sizeof(vk_op_ssm_conv_push_constants), {32, 16, 1}, {32, 16, 0, 1}, 1); - ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_bias_silu_f32, "ssm_conv_bias_silu_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 4, sizeof(vk_op_ssm_conv_push_constants), {32, 16, 1}, {32, 16, 1, 1}, 1); + if (device->coopmat_support) { + device_extensions.push_back("VK_KHR_cooperative_matrix"); + } +#endif + device->name = GGML_VK_NAME + std::to_string(idx); - ggml_vk_create_pipeline(device, device->pipeline_opt_step_adamw_f32, "opt_step_adamw_f32", opt_step_adamw_f32_len, opt_step_adamw_f32_data, "main", 5, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + device_create_info + .setFlags(vk::DeviceCreateFlags()) + .setQueueCreateInfos(device_queue_create_infos) + .setPEnabledExtensionNames(device_extensions); + device_create_info.setPNext(&device_features2); + device->device = device->physical_device.createDevice(device_create_info); - ggml_vk_create_pipeline(device, device->pipeline_opt_step_sgd_f32, "opt_step_sgd_f32", opt_step_sgd_f32_len, opt_step_sgd_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + if (device->device_fault) { + device->pfn_vkGetDeviceFaultInfoEXT = (PFN_vkGetDeviceFaultInfoEXT) + vkGetDeviceProcAddr(device->device, "vkGetDeviceFaultInfoEXT"); + } - // conv2d, conv_transpose_2d, conv3d - for (uint32_t s = 0; s < CONV_SHAPE_COUNT; ++s) { - // smaller WG for the small-tile fallback gives more concurrent WGs per SM - uint32_t conv2d_WG_SIZE = (s == CONV_SHAPE_64x32) ? 128 : 256; - uint32_t use_collectives = 0; // Enables subgroup ops for preventing the re-calculation of indices. - uint32_t conv2d_TS_K = (s == CONV_SHAPE_64x32) ? 4 : 8; - uint32_t conv2d_SHMEM_PAD = 4; - vk_conv_block_size conv2d_BS = vk_conv_block_sizes[s]; - bool conv2d_UNROLL = true; + // Queues + device->compute_queue = ggml_vk_create_queue(device, compute_queue_family_index, 0, { vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer }, false); -#if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) - if (device->coopmat2) { - conv2d_SHMEM_PAD = 8; // 8 float16_t - } + // Shaders + // Disable matmul tile sizes early if performance low or not supported + for (uint32_t i = 0; i < GGML_TYPE_COUNT; ++i) { + switch (device->vendor_id) { +#ifndef GGML_VULKAN_RUN_TESTS + case VK_VENDOR_ID_AMD: + device->mul_mat_l[i] = device->coopmat_support && device->driver_id != vk::DriverId::eAmdProprietary; + device->mul_mat_m[i] = true; + device->mul_mat_s[i] = true; + device->mul_mat_id_l[i] = false; + device->mul_mat_id_m[i] = true; + device->mul_mat_id_s[i] = true; + break; + case VK_VENDOR_ID_INTEL: { + // Current Windows driver does not expose BF16 support. + // We only want to use l_warptile if coopmat is available + const bool use_l_warptile = (i == GGML_TYPE_BF16) ? (device->coopmat_bf16_support && device->coopmat_support) : device->coopmat_support; + device->mul_mat_l[i] = use_l_warptile; + device->mul_mat_id_l[i] = use_l_warptile; + device->mul_mat_m[i] = true; + device->mul_mat_s[i] = true; + device->mul_mat_id_m[i] = true; + device->mul_mat_id_s[i] = true; + break; + } + case VK_VENDOR_ID_APPLE: + device->mul_mat_l[i] = false; + device->mul_mat_m[i] = true; + device->mul_mat_s[i] = false; + device->mul_mat_id_l[i] = false; + device->mul_mat_id_m[i] = true; + device->mul_mat_id_s[i] = false; + break; + case VK_VENDOR_ID_QUALCOMM: + device->mul_mat_l[i] = false; + device->mul_mat_m[i] = true; + device->mul_mat_s[i] = true; + device->mul_mat_id_l[i] = false; + device->mul_mat_id_m[i] = true; + device->mul_mat_id_s[i] = true; + break; #endif + default: + device->mul_mat_l[i] = true; + device->mul_mat_m[i] = true; + device->mul_mat_s[i] = true; + device->mul_mat_id_l[i] = true; + device->mul_mat_id_m[i] = true; + device->mul_mat_id_s[i] = true; + break; + } - if (device->vendor_id == VK_VENDOR_ID_INTEL) { - conv2d_SHMEM_PAD = 0; - conv2d_UNROLL = false; - } else if (device->vendor_id == VK_VENDOR_ID_AMD) { - conv2d_SHMEM_PAD = device->architecture == vk_device_architecture::AMD_GCN ? 1 : 4; - if (s == CONV_SHAPE_128x128 && device->architecture != vk_device_architecture::AMD_GCN) { - conv2d_UNROLL = false; +#if VK_HEADER_VERSION >= 287 + // Honeykrisp driver for Asahi Linux doesn't report VK_VENDOR_ID_APPLE. + // Check for Honeykrisp driver and force same configuration as the VK_VENDOR_ID_APPLE case. + if (device->driver_id == vk::DriverId::eMesaHoneykrisp) { + device->mul_mat_l[i] = false; + device->mul_mat_m[i] = true; + device->mul_mat_s[i] = false; + device->mul_mat_id_l[i] = false; + device->mul_mat_id_m[i] = true; + device->mul_mat_id_s[i] = false; } +#endif + + device->mul_mat_l_int[i] = device->mul_mat_l[i]; + device->mul_mat_m_int[i] = device->mul_mat_m[i]; + device->mul_mat_s_int[i] = device->mul_mat_s[i]; + device->mul_mat_id_l_int[i] = device->mul_mat_id_l[i]; + device->mul_mat_id_m_int[i] = device->mul_mat_id_m[i]; + device->mul_mat_id_s_int[i] = device->mul_mat_id_s[i]; } - // Use collectives on pre-Turing NVIDIA GPUs and GCN AMD cards, which had slower integer math. - bool allow_collectives_nv = device->vendor_id != VK_VENDOR_ID_NVIDIA || - device->architecture == vk_device_architecture::NVIDIA_PRE_TURING; - bool allow_collectives_amd = device->vendor_id != VK_VENDOR_ID_AMD || - device->architecture == vk_device_architecture::AMD_GCN; - if (device->subgroup_shuffle && - device->vendor_id != VK_VENDOR_ID_INTEL && // Do not enable collectives on Intel, see PR 14316. - allow_collectives_nv && - allow_collectives_amd) { - use_collectives = 1; - conv2d_BS.CRS = std::min( - device->subgroup_size, - conv2d_BS.CRS); // CRS block size should be capped at subgroup size for correctness when shuffle is used. + std::vector<vk::DescriptorSetLayoutBinding> dsl_binding; + std::vector<vk::DescriptorBindingFlags> dsl_binding_flags; + for (uint32_t i = 0; i < MAX_PARAMETER_COUNT; i++) { + dsl_binding.push_back({i, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute}); + dsl_binding_flags.push_back({}); } - // cm1 is used only when cm2 is unavailable; capped at 64x128 (due to shared memory size). - // Requires 16x16x16 f16-acc since that's the fragment shape hard-coded in the shader. - // Subgroup size must be 32 or 64 (to keep WG_SIZE sane) and we need - // subgroup_size_control to force the driver to actually use it. - bool conv2d_use_cm1 = false; -#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - conv2d_use_cm1 = !device->coopmat2 && - device->coopmat_support && device->coopmat_support_16x16x16_f16acc && - device->subgroup_size_control && - (device->subgroup_size == 32 || device->subgroup_size == 64) && - s != CONV_SHAPE_128x128; -#endif - - const uint32_t conv2d_cm1_shmem_pad = 8; + vk::DescriptorSetLayoutBindingFlagsCreateInfo dslbfci = { dsl_binding_flags }; - auto shmem_req = [&](uint32_t pad, bool csh_store, bool fp16_shmem) { - const uint32_t elem_size = fp16_shmem ? (uint32_t)sizeof(uint16_t) : (uint32_t)sizeof(float); - const uint32_t csh_elems = csh_store ? conv2d_BS.K * conv2d_BS.NPQ : 0u; - return (conv2d_BS.K * (conv2d_BS.CRS + pad) + conv2d_BS.CRS * (conv2d_BS.NPQ + pad) + csh_elems) * elem_size; - }; + vk::DescriptorSetLayoutCreateInfo descriptor_set_layout_create_info( + {}, + dsl_binding); + descriptor_set_layout_create_info.setPNext(&dslbfci); + device->dsl = device->device.createDescriptorSetLayout(descriptor_set_layout_create_info); - // 2D, transpose-2D, and 3D conv use the same KxCRS @ CRSxNPQ shmem - // layout. cm1 needs Csh for output, so check before applying cm1 params. - if (conv2d_use_cm1 && device->properties.limits.maxComputeSharedMemorySize < shmem_req(conv2d_cm1_shmem_pad, true, true)) { - conv2d_use_cm1 = false; - } + ggml_vk_load_shaders(device); + GGML_LOG_INFO("ggml_vulkan: %s multi_add=%d rte_fp16=%d\n", + device->name.c_str(), (int) device->multi_add, + (int) device->float_controls_rte_fp16); - uint32_t conv2d_WM = 16, conv2d_WN = 16; // cm1 subgroup tile, ignored otherwise - if (conv2d_use_cm1) { - conv2d_SHMEM_PAD = conv2d_cm1_shmem_pad; - // 16x16x16 fragments; pick WM/WN to keep WG_SIZE at 256 - // (i.e. 8 subgroups for sg=32, 4 subgroups for sg=64). - const bool sg64 = (device->subgroup_size == 64); - switch (s) { - case CONV_SHAPE_64x32: conv2d_WM = sg64 ? 32 : 16; conv2d_WN = 16; break; - case CONV_SHAPE_64x128: conv2d_WM = 32; conv2d_WN = sg64 ? 64 : 32; break; - case CONV_SHAPE_32x256: conv2d_WM = sg64 ? 16 : 32; conv2d_WN = sg64 ? 128 : 32; break; - default: break; - } - const uint32_t warps_M = conv2d_BS.K / conv2d_WM; - const uint32_t warps_N = conv2d_BS.NPQ / conv2d_WN; - conv2d_WG_SIZE = warps_M * warps_N * device->subgroup_size; - } + // Prefer a dedicated transfer queue on AMD dGPUs (non-GCN) when graphics queue use is disabled. + const bool prefers_transfer_queue = + device->vendor_id == VK_VENDOR_ID_AMD && + device->architecture != AMD_GCN && + !device->uma && + !allow_graphics_queue; - // stage cm2 accumulator through shmem for coalesced global stores; - // skipped on 128x128 where the extra Csh footprint hurts occupancy. - // cm1 always uses the staged path. - uint32_t conv2d_csh_store = (device->coopmat2 && s != CONV_SHAPE_128x128) ? 1u : 0u; - if (conv2d_use_cm1) { - conv2d_csh_store = 1; - } + if (!device->single_queue) { + const uint32_t transfer_queue_index = compute_queue_family_index == transfer_queue_family_index ? 1 : 0; + device->transfer_queue = ggml_vk_create_queue(device, transfer_queue_family_index, transfer_queue_index, { vk::PipelineStageFlagBits::eTransfer }, true); - // shmem is fp16 on cm2/cm1 (matches Csh), fp32 on scalar - const bool conv2d_use_fp16_shmem = device->coopmat2 || conv2d_use_cm1; + device->async_use_transfer_queue = prefers_transfer_queue || (getenv("GGML_VK_ASYNC_USE_TRANSFER_QUEUE") != nullptr); + } else { + device->transfer_queue = ggml_vk_create_aliased_queue(device, device->compute_queue); - // shrink CRS if the non-cm1 config still doesn't fit - if (device->properties.limits.maxComputeSharedMemorySize < shmem_req(conv2d_SHMEM_PAD, conv2d_csh_store, conv2d_use_fp16_shmem)) { - GGML_ASSERT(!conv2d_use_cm1); - conv2d_BS.CRS = 8; - if (use_collectives) { - conv2d_BS.CRS = std::min(device->subgroup_size, conv2d_BS.CRS); - } - conv2d_csh_store = 0; + device->async_use_transfer_queue = false; } - std::array<uint32_t, 3> wg_denoms = { conv2d_BS.K, 1, 1 }; - std::vector<uint32_t> spec_constants = { conv2d_WG_SIZE, conv2d_BS.K, conv2d_BS.CRS, conv2d_BS.NPQ, conv2d_TS_K, use_collectives, conv2d_SHMEM_PAD }; + device->buffer_type = { + /* .iface = */ ggml_backend_vk_buffer_type_interface, + /* .device = */ ggml_backend_reg_dev_get(ggml_backend_vk_reg(), idx), + /* .context = */ new ggml_backend_vk_buffer_type_context{ device->name, device }, + }; - // cm1 needs a fixed subgroup width to match the WG_SIZE we computed - const uint32_t conv2d_required_subgroup_size = conv2d_use_cm1 ? device->subgroup_size : 0; + device->fence = device->device.createFence({}); -#define CREATE_CONV(name, type_suffix, spv_suffix) \ - for (auto &c : device->pipeline_##name##type_suffix[s]) { \ - const vk_conv2d_pipeline_state &state = c.first; \ - std::vector<uint32_t> spec_constants_cpy = spec_constants; \ - spec_constants_cpy.push_back(state.s0); \ - spec_constants_cpy.push_back(state.s1); \ - spec_constants_cpy.push_back(state.p0); \ - spec_constants_cpy.push_back(state.p1); \ - spec_constants_cpy.push_back(state.d0); \ - spec_constants_cpy.push_back(state.d1); \ - spec_constants_cpy.push_back(state.KW); \ - spec_constants_cpy.push_back(state.KH); \ - spec_constants_cpy.push_back(state.aligned); \ - spec_constants_cpy.push_back(conv2d_csh_store); \ - spec_constants_cpy.push_back(conv2d_WM); \ - spec_constants_cpy.push_back(conv2d_WN); \ - ggml_vk_create_pipeline( \ - device, c.second, #name #type_suffix, \ - name##type_suffix##spv_suffix##_len, name##type_suffix##spv_suffix##_data, "main", 3, \ - sizeof(vk_op_conv2d_push_constants), wg_denoms, spec_constants_cpy, 1, true, use_collectives || conv2d_required_subgroup_size, conv2d_required_subgroup_size); \ - } -#define CREATE_CONVS(spv_suffix) \ - CREATE_CONV(conv2d, _f32, spv_suffix) \ - CREATE_CONV(conv2d, _f16_f32, spv_suffix) \ - CREATE_CONV(conv_transpose_2d, _f32, spv_suffix) \ - CREATE_CONV(conv_transpose_2d, _f16_f32, spv_suffix) -#if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) - if (device->coopmat2) { - CREATE_CONVS(_cm2) - } else -#endif -#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - if (conv2d_use_cm1) { - CREATE_CONVS(_cm1) - } else -#endif - if (conv2d_UNROLL) { - CREATE_CONVS(_unroll) - } else { - CREATE_CONVS( ) - } -#undef CREATE_CONV -#undef CREATE_CONVS + device->idx = idx; - std::vector<uint32_t> conv3d_spec_constants = { conv2d_WG_SIZE, conv2d_BS.K, conv2d_BS.CRS, conv2d_BS.NPQ, conv2d_TS_K, conv2d_SHMEM_PAD }; -#define CREATE_CONV3D(type_suffix, spv_suffix) \ - for (auto &c : device->pipeline_conv3d##type_suffix[s]) { \ - const vk_conv3d_pipeline_state &state = c.first; \ - std::vector<uint32_t> spec_constants_cpy = conv3d_spec_constants; \ - spec_constants_cpy.push_back(state.s0); \ - spec_constants_cpy.push_back(state.s1); \ - spec_constants_cpy.push_back(state.s2); \ - spec_constants_cpy.push_back(state.p0); \ - spec_constants_cpy.push_back(state.p1); \ - spec_constants_cpy.push_back(state.p2); \ - spec_constants_cpy.push_back(state.d0); \ - spec_constants_cpy.push_back(state.d1); \ - spec_constants_cpy.push_back(state.d2); \ - spec_constants_cpy.push_back(state.KW); \ - spec_constants_cpy.push_back(state.KH); \ - spec_constants_cpy.push_back(state.KD); \ - spec_constants_cpy.push_back(state.aligned); \ - spec_constants_cpy.push_back(conv2d_csh_store); \ - spec_constants_cpy.push_back(conv2d_WM); \ - spec_constants_cpy.push_back(conv2d_WN); \ - ggml_vk_create_pipeline( \ - device, c.second, "conv3d" #type_suffix, \ - conv3d##type_suffix##spv_suffix##_len, conv3d##type_suffix##spv_suffix##_data, "main", 3, \ - sizeof(vk_op_conv3d_push_constants), wg_denoms, spec_constants_cpy, 1, true, conv2d_required_subgroup_size != 0, conv2d_required_subgroup_size); \ - } -#if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) - if (device->coopmat2) { - CREATE_CONV3D(_f32, _cm2) - CREATE_CONV3D(_f16_f32, _cm2) - } else -#endif -#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - if (conv2d_use_cm1) { - CREATE_CONV3D(_f32, _cm1) - CREATE_CONV3D(_f16_f32, _cm1) - } else -#endif - if (conv2d_UNROLL) { - CREATE_CONV3D(_f32, _unroll) - CREATE_CONV3D(_f16_f32, _unroll) - } else { - CREATE_CONV3D(_f32, ) - CREATE_CONV3D(_f16_f32, ) - } -#undef CREATE_CONV3D - } + device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr; - ggml_vk_create_pipeline(device, device->pipeline_conv2d_dw_whcn_f32, "conv2d_dw_whcn_f32", conv2d_dw_whcn_f32_len, conv2d_dw_whcn_f32_data, "main", 3, sizeof(vk_op_conv2d_dw_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_conv2d_dw_cwhn_f32, "conv2d_dw_cwhn_f32", conv2d_dw_cwhn_f32_len, conv2d_dw_cwhn_f32_data, "main", 3, sizeof(vk_op_conv2d_dw_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_conv2d_dw_whcn_f16_f32, "conv2d_dw_whcn_f16_f32", conv2d_dw_whcn_f16_f32_len, conv2d_dw_whcn_f16_f32_data, "main", 3, sizeof(vk_op_conv2d_dw_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_conv2d_dw_cwhn_f16_f32, "conv2d_dw_cwhn_f16_f32", conv2d_dw_cwhn_f16_f32_len, conv2d_dw_cwhn_f16_f32_data, "main", 3, sizeof(vk_op_conv2d_dw_push_constants), {512, 1, 1}, {}, 1); + device->disable_fusion = getenv("GGML_VK_DISABLE_FUSION") != nullptr; - for (uint32_t use_push = 0; use_push < 2; ++use_push) { - for (uint32_t i = 0; i < num_topk_moe_pipelines; ++i) { - ggml_vk_create_pipeline2(device, device->pipeline_topk_moe[i][use_push], "topk_moe_f32_"+std::to_string(i), topk_moe_f32_len, topk_moe_f32_data, "main", 4, sizeof(vk_op_topk_moe_push_constants), {1, 1, 1}, {device->subgroup_size, 1u<<i, use_push}, 1, true, true, device->subgroup_size); + device->add_rms_fusion = !device->disable_fusion && + device->subgroup_arithmetic && + device->vendor_id != VK_VENDOR_ID_INTEL; + device->partials_binding_alignment = + std::max(4u, (uint32_t)device->properties.limits.minStorageBufferOffsetAlignment); + + device->mmvq_mode = 0; + if (getenv("GGML_VK_DISABLE_MMVQ")) { + device->mmvq_mode = -1; + } else if (getenv("GGML_VK_FORCE_MMVQ")) { + device->mmvq_mode = 1; } + + return device; } - ggml_vk_create_pipeline(device, device->pipeline_wp_fused_expert_q5_1, - "wp_fused_expert_q5_1", wp_fused_expert_q5_1_len, - wp_fused_expert_q5_1_data, "main", 6, - sizeof(vk_wp_fused_expert_push_constants), {1, 1, 1}, {}, 1, - true, true, 64); - ggml_vk_create_pipeline(device, device->pipeline_wp_fused_expert_q8_0, - "wp_fused_expert_q8_0", wp_fused_expert_q8_0_len, - wp_fused_expert_q8_0_data, "main", 6, - sizeof(vk_wp_fused_expert_push_constants), {1, 1, 1}, {}, 1, - true, true, 64); - // Phase A: 5 bindings -- gate, up, input, params, scratch (no down - // buffer, no output buffer; it only ever writes the scratch buffer). - ggml_vk_create_pipeline(device, device->pipeline_wp_fused_expert_batch_a, - "wp_fused_expert_batch_a", wp_fused_expert_batch_a_len, - wp_fused_expert_batch_a_data, "main", 5, - sizeof(vk_wp_fused_expert_batch_push_constants), {1, 1, 1}, {}, 1, - true, true, 64); - // Phase B: 4 bindings -- down, output, params, scratch (no gate/up/input - // buffer; it only ever reads the scratch buffer phase A wrote). - ggml_vk_create_pipeline(device, device->pipeline_wp_fused_expert_batch_b_q5_1, - "wp_fused_expert_batch_b_q5_1", wp_fused_expert_batch_b_q5_1_len, - wp_fused_expert_batch_b_q5_1_data, "main", 4, - sizeof(vk_wp_fused_expert_batch_push_constants), {1, 1, 1}, {}, 1, - true, true, 64); - ggml_vk_create_pipeline(device, device->pipeline_wp_fused_expert_batch_b_q8_0, - "wp_fused_expert_batch_b_q8_0", wp_fused_expert_batch_b_q8_0_len, - wp_fused_expert_batch_b_q8_0_data, "main", 4, - sizeof(vk_wp_fused_expert_batch_push_constants), {1, 1, 1}, {}, 1, - true, true, 64); + return vk_instance.devices[idx]; +} - // Drop compile_mutex so other threads can walk while we compile. - compile_lock.unlock(); +static void ggml_vk_print_gpu_info(size_t idx) { + GGML_ASSERT(idx < vk_instance.device_indices.size()); + size_t dev_num = vk_instance.device_indices[idx]; + VK_LOG_DEBUG("ggml_vk_print_gpu_info(" << dev_num << ")"); + GGML_ASSERT(vk_instance_initialized); - // Compile what we claimed; create_pipeline_func reacquires compile_mutex - // at the end to flip compile_pending/compiled and notify waiters. - if (has_claimed_task) { - auto & task = claimed_task; - ggml_vk_create_pipeline_func(device, task.pipeline, task.spv_size, task.spv_data, - task.entrypoint, task.parameter_count, task.wg_denoms, - task.specialization_constants, task.disable_robustness, - task.require_full_subgroups, task.required_subgroup_size); - } + std::vector<vk::PhysicalDevice> devices = vk_instance.instance.enumeratePhysicalDevices(); - // Another thread may be compiling the pipeline we need; block on it here. - if (wait_pipeline) { - std::unique_lock<std::mutex> wait_lock(device->compile_mutex); - device->compile_cv.wait(wait_lock, [&] { - return wait_pipeline->compiled.load(); - }); + if (dev_num >= devices.size()) { + std::cerr << "ggml_vulkan: Device with index " << dev_num << " does not exist." << std::endl; + throw std::runtime_error("Device not found"); } -} - -static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch); -static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev); - -static vk_device ggml_vk_get_device(size_t idx) { - VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")"); - - if (vk_instance.devices[idx] == nullptr) { - VK_LOG_DEBUG("Initializing new vk_device"); - vk_device device = std::make_shared<vk_device_struct>(); - vk_instance.devices[idx] = device; - - device->memory_logger = std::unique_ptr<vk_memory_logger>(new vk_memory_logger()); - - size_t dev_num = vk_instance.device_indices[idx]; - - std::vector<vk::PhysicalDevice> physical_devices = vk_instance.instance.enumeratePhysicalDevices(); - if (dev_num >= physical_devices.size()) { - std::cerr << "ggml_vulkan: Device with index " << dev_num << " does not exist." << std::endl; - throw std::runtime_error("Device not found"); - } - - device->physical_device = physical_devices[dev_num]; - const std::vector<vk::ExtensionProperties> ext_props = device->physical_device.enumerateDeviceExtensionProperties(); - - device->architecture = get_device_architecture(device->physical_device); - - const char* GGML_VK_PREFER_HOST_MEMORY = getenv("GGML_VK_PREFER_HOST_MEMORY"); - device->prefer_host_memory = GGML_VK_PREFER_HOST_MEMORY != nullptr; - - const char* GGML_VK_DISABLE_HOST_VISIBLE_VIDMEM = getenv("GGML_VK_DISABLE_HOST_VISIBLE_VIDMEM"); - device->disable_host_visible_vidmem = GGML_VK_DISABLE_HOST_VISIBLE_VIDMEM != nullptr; - - const char* GGML_VK_ALLOW_SYSMEM_FALLBACK = getenv("GGML_VK_ALLOW_SYSMEM_FALLBACK"); - device->allow_sysmem_fallback = GGML_VK_ALLOW_SYSMEM_FALLBACK != nullptr; - - const char* GGML_VK_DISABLE_GRAPH_OPTIMIZE = getenv("GGML_VK_DISABLE_GRAPH_OPTIMIZE"); - device->disable_graph_optimize = GGML_VK_DISABLE_GRAPH_OPTIMIZE != nullptr; + vk::PhysicalDevice physical_device = devices[dev_num]; + std::vector<vk::ExtensionProperties> ext_props = physical_device.enumerateDeviceExtensionProperties(); - bool fp16_storage = false; - bool fp16_compute = false; - bool maintenance4_support = false; - bool sm_builtins = false; - bool amd_shader_core_properties2 = false; - bool pipeline_robustness = false; - bool coopmat2_support = false; - bool coopmat2_decode_vector_support = false; - bool pipeline_executable_properties_support = false; - bool internally_sync_support = false; - device->coopmat_support = false; - device->integer_dot_product = false; - device->shader_64b_indexing = false; - bool bfloat16_support = false; - bool dot2_f16_support = false; - bool ocp_microscaling_extension = false; - bool shader_float8_extension = false; + bool fp16_storage = false; + bool fp16_compute = false; + bool coopmat_support = false; + bool coopmat2_support = false; + bool coopmat2_decode_vector_support = false; + bool integer_dot_product = false; + bool bfloat16_support = false; + bool dot2_f16_support = false; + bool ocp_microscaling_extension = false; + bool shader_float8_extension = false; - for (const auto& properties : ext_props) { - if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) { - maintenance4_support = true; - } else if (strcmp("VK_KHR_16bit_storage", properties.extensionName) == 0) { - fp16_storage = true; - } else if (strcmp("VK_KHR_shader_float16_int8", properties.extensionName) == 0) { - fp16_compute = true; - } else if (strcmp("VK_NV_shader_sm_builtins", properties.extensionName) == 0) { - sm_builtins = true; - } else if (strcmp("VK_AMD_shader_core_properties2", properties.extensionName) == 0) { - amd_shader_core_properties2 = true; - } else if (strcmp("VK_EXT_pipeline_robustness", properties.extensionName) == 0) { - pipeline_robustness = true; - } else if (strcmp("VK_EXT_subgroup_size_control", properties.extensionName) == 0) { - device->subgroup_size_control = true; + for (auto properties : ext_props) { + if (strcmp("VK_KHR_16bit_storage", properties.extensionName) == 0) { + fp16_storage = true; + } else if (strcmp("VK_KHR_shader_float16_int8", properties.extensionName) == 0) { + fp16_compute = true; #if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - } else if (strcmp("VK_KHR_cooperative_matrix", properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_COOPMAT")) { - device->coopmat_support = true; - device->coopmat_m = 0; - device->coopmat_n = 0; - device->coopmat_k = 0; + } else if (strcmp("VK_KHR_cooperative_matrix", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_COOPMAT")) { + coopmat_support = true; #endif #if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) - } else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_COOPMAT2")) { - coopmat2_support = true; + } else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_COOPMAT2")) { + coopmat2_support = true; #endif - } else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) { - coopmat2_decode_vector_support = true; + } else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) { + coopmat2_decode_vector_support = true; #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - } else if (strcmp("VK_KHR_shader_integer_dot_product", properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_INTEGER_DOT_PRODUCT")) { - device->integer_dot_product = true; + } else if (strcmp("VK_KHR_shader_integer_dot_product", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_INTEGER_DOT_PRODUCT")) { + integer_dot_product = true; #endif #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - } else if (strcmp("VK_KHR_shader_bfloat16", properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_BFLOAT16")) { - bfloat16_support = true; + } else if (strcmp("VK_KHR_shader_bfloat16", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_BFLOAT16")) { + bfloat16_support = true; #endif #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) - } else if (strcmp(VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME, properties.extensionName) == 0) { - ocp_microscaling_extension = true; + } else if (strcmp(VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME, properties.extensionName) == 0) { + ocp_microscaling_extension = true; #endif #if defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - } else if (strcmp(VK_EXT_SHADER_FLOAT8_EXTENSION_NAME, properties.extensionName) == 0) { - shader_float8_extension = true; -#endif - } else if (strcmp("VK_VALVE_shader_mixed_float_dot_product", properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_DOT2")) { - dot2_f16_support = true; - } else if (strcmp("VK_KHR_pipeline_executable_properties", properties.extensionName) == 0) { - pipeline_executable_properties_support = true; - } else if (strcmp("VK_EXT_memory_priority", properties.extensionName) == 0 && - getenv("GGML_VK_ENABLE_MEMORY_PRIORITY")) { - device->memory_priority = true; - } else if (strcmp("VK_EXT_external_memory_host", properties.extensionName) == 0) { - device->external_memory_host = true; - } else if (strcmp("VK_KHR_external_semaphore_fd", properties.extensionName) == 0) { - device->external_semaphore_fd = true; -#if defined(VK_EXT_shader_64bit_indexing) - } else if (strcmp("VK_EXT_shader_64bit_indexing", properties.extensionName) == 0) { - device->shader_64b_indexing = true; + } else if (strcmp(VK_EXT_SHADER_FLOAT8_EXTENSION_NAME, properties.extensionName) == 0) { + shader_float8_extension = true; #endif - } else if (strcmp(VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME, properties.extensionName) == 0) { - internally_sync_support = true; - } else if (strcmp("VK_EXT_device_fault", properties.extensionName) == 0) { - device->device_fault = true; - } + } else if (strcmp("VK_VALVE_shader_mixed_float_dot_product", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_DOT2")) { + dot2_f16_support = true; } + } - vk::PhysicalDeviceProperties2 props2; - vk::PhysicalDeviceMaintenance3Properties props3; - vk::PhysicalDeviceMaintenance4Properties props4; - vk::PhysicalDeviceSubgroupProperties subgroup_props; - vk::PhysicalDeviceDriverProperties driver_props; - vk::PhysicalDeviceShaderSMBuiltinsPropertiesNV sm_props; - vk::PhysicalDeviceShaderCoreProperties2AMD amd_shader_core_properties2_props; - vk::PhysicalDeviceVulkan11Properties vk11_props; - vk::PhysicalDeviceVulkan12Properties vk12_props; - vk::PhysicalDeviceSubgroupSizeControlPropertiesEXT subgroup_size_control_props; - vk::PhysicalDeviceShaderIntegerDotProductPropertiesKHR shader_integer_dot_product_props; - vk::PhysicalDeviceExternalMemoryHostPropertiesEXT external_memory_host_props; + const vk_device_architecture device_architecture = get_device_architecture(physical_device); - props2.pNext = &props3; - props3.pNext = &subgroup_props; - subgroup_props.pNext = &driver_props; - driver_props.pNext = &vk11_props; - vk11_props.pNext = &vk12_props; + const char* GGML_VK_DISABLE_F16 = getenv("GGML_VK_DISABLE_F16"); + bool force_disable_f16 = GGML_VK_DISABLE_F16 != nullptr; - VkBaseOutStructure * last_struct = (VkBaseOutStructure *)&vk12_props; + bool fp16 = !force_disable_f16 && fp16_storage && fp16_compute; - if (maintenance4_support) { - last_struct->pNext = (VkBaseOutStructure *)&props4; - last_struct = (VkBaseOutStructure *)&props4; - } - if (sm_builtins) { - last_struct->pNext = (VkBaseOutStructure *)&sm_props; - last_struct = (VkBaseOutStructure *)&sm_props; - } - if (amd_shader_core_properties2) { - last_struct->pNext = (VkBaseOutStructure *)&amd_shader_core_properties2_props; - last_struct = (VkBaseOutStructure *)&amd_shader_core_properties2_props; - } - if (device->subgroup_size_control) { - last_struct->pNext = (VkBaseOutStructure *)&subgroup_size_control_props; - last_struct = (VkBaseOutStructure *)&subgroup_size_control_props; - } + vk::PhysicalDeviceProperties2 props2; + vk::PhysicalDeviceMaintenance3Properties props3; + vk::PhysicalDeviceSubgroupProperties subgroup_props; + vk::PhysicalDeviceDriverProperties driver_props; + vk::PhysicalDeviceShaderIntegerDotProductPropertiesKHR shader_integer_dot_product_props; + props2.pNext = &props3; + props3.pNext = &subgroup_props; + subgroup_props.pNext = &driver_props; -#if defined(VK_NV_cooperative_matrix2) - vk::PhysicalDeviceCooperativeMatrix2PropertiesNV coopmat2_props; - if (coopmat2_support) { - last_struct->pNext = (VkBaseOutStructure *)&coopmat2_props; - last_struct = (VkBaseOutStructure *)&coopmat2_props; - } -#endif + // Pointer to the last chain element + VkBaseOutStructure * last_struct = (VkBaseOutStructure *)&driver_props; - if (device->integer_dot_product) { - last_struct->pNext = (VkBaseOutStructure *)&shader_integer_dot_product_props; - last_struct = (VkBaseOutStructure *)&shader_integer_dot_product_props; - } + if (integer_dot_product) { + last_struct->pNext = (VkBaseOutStructure *)&shader_integer_dot_product_props; + last_struct = (VkBaseOutStructure *)&shader_integer_dot_product_props; + } - if (device->external_memory_host) { - last_struct->pNext = (VkBaseOutStructure *)&external_memory_host_props; - last_struct = (VkBaseOutStructure *)&external_memory_host_props; - } + physical_device.getProperties2(&props2); - device->physical_device.getProperties2(&props2); - device->properties = props2.properties; - device->vendor_id = device->properties.vendorID; - device->driver_id = driver_props.driverID; + VkPhysicalDeviceFeatures2 device_features2; + device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + device_features2.pNext = nullptr; - if (device->driver_id == vk::DriverId::eMoltenvk) { - // Disable external_memory_host until https://github.com/KhronosGroup/MoltenVK/pull/2622 - // is available in the Vulkan SDK. - device->external_memory_host = false; - } + VkPhysicalDeviceVulkan11Features vk11_features; + vk11_features.pNext = nullptr; + vk11_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES; + device_features2.pNext = &vk11_features; - // Implementing the async backend interfaces seems broken on older Intel HW, - // see https://github.com/ggml-org/llama.cpp/issues/17302. - device->support_async = (device->vendor_id != VK_VENDOR_ID_INTEL || - std::string(device->properties.deviceName.data()).find("(DG1)") == std::string::npos) && - getenv("GGML_VK_DISABLE_ASYNC") == nullptr; + VkPhysicalDeviceVulkan12Features vk12_features; + vk12_features.pNext = nullptr; + vk12_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; + vk11_features.pNext = &vk12_features; - if (!device->support_async) { - GGML_LOG_DEBUG("ggml_vulkan: WARNING: Async execution disabled on certain Intel devices.\n"); - } + // Pointer to the last chain element + last_struct = (VkBaseOutStructure *)&vk12_features; - const char* GGML_VK_FORCE_MAX_ALLOCATION_SIZE = getenv("GGML_VK_FORCE_MAX_ALLOCATION_SIZE"); +#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + VkPhysicalDeviceCooperativeMatrixFeaturesKHR coopmat_features; + coopmat_features.pNext = nullptr; + coopmat_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR; + coopmat_features.cooperativeMatrix = VK_FALSE; - if (GGML_VK_FORCE_MAX_ALLOCATION_SIZE != nullptr) { - device->max_memory_allocation_size = std::stoull(GGML_VK_FORCE_MAX_ALLOCATION_SIZE); - } else if (maintenance4_support) { - device->max_memory_allocation_size = std::min(props3.maxMemoryAllocationSize, props4.maxBufferSize); - } else { - device->max_memory_allocation_size = props3.maxMemoryAllocationSize; - } + if (coopmat_support) { + last_struct->pNext = (VkBaseOutStructure *)&coopmat_features; + last_struct = (VkBaseOutStructure *)&coopmat_features; + } +#endif - const char* GGML_VK_FORCE_MAX_BUFFER_SIZE = getenv("GGML_VK_FORCE_MAX_BUFFER_SIZE"); + VkPhysicalDeviceShaderIntegerDotProductFeaturesKHR shader_integer_dot_product_features {}; + shader_integer_dot_product_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_INTEGER_DOT_PRODUCT_FEATURES_KHR; + if (integer_dot_product) { + last_struct->pNext = (VkBaseOutStructure *)&shader_integer_dot_product_features; + last_struct = (VkBaseOutStructure *)&shader_integer_dot_product_features; + } - if (GGML_VK_FORCE_MAX_BUFFER_SIZE != nullptr) { - device->max_buffer_size = std::stoull(GGML_VK_FORCE_MAX_BUFFER_SIZE); - } else if (maintenance4_support) { - device->max_buffer_size = props4.maxBufferSize; - } else { - device->max_buffer_size = device->max_memory_allocation_size; - } +#if defined(VK_KHR_shader_bfloat16) + VkPhysicalDeviceShaderBfloat16FeaturesKHR bfloat16_features {}; + bfloat16_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR; + if (bfloat16_support) { + last_struct->pNext = (VkBaseOutStructure *)&bfloat16_features; + last_struct = (VkBaseOutStructure *)&bfloat16_features; + } +#endif - const char* GGML_VK_SUBALLOCATION_BLOCK_SIZE = getenv("GGML_VK_SUBALLOCATION_BLOCK_SIZE"); +#if defined(VK_NV_cooperative_matrix2) + VkPhysicalDeviceCooperativeMatrix2FeaturesNV coopmat2_features {}; + coopmat2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_2_FEATURES_NV; + if (coopmat2_support) { + last_struct->pNext = (VkBaseOutStructure *)&coopmat2_features; + last_struct = (VkBaseOutStructure *)&coopmat2_features; + } +#endif - if (GGML_VK_SUBALLOCATION_BLOCK_SIZE != nullptr) { - device->suballocation_block_size = std::stoull(GGML_VK_SUBALLOCATION_BLOCK_SIZE); - } else { - // Limit batching of allocations to 1GB by default to avoid fragmentation issues - device->suballocation_block_size = 1024*1024*1024; - } - device->suballocation_block_size = std::min(device->suballocation_block_size, device->max_memory_allocation_size); + VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV coopmat2_decode_vector_features {}; + coopmat2_decode_vector_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV; + if (coopmat2_decode_vector_support) { + last_struct->pNext = (VkBaseOutStructure *)&coopmat2_decode_vector_features; + last_struct = (VkBaseOutStructure *)&coopmat2_decode_vector_features; + } - device->subgroup_size = subgroup_props.subgroupSize; - device->subgroup_size_log2 = uint32_t(log2f(float(device->subgroup_size))); - device->uma = device->properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; - if (sm_builtins) { - device->shader_core_count = sm_props.shaderSMCount; - } else if (amd_shader_core_properties2) { - device->shader_core_count = amd_shader_core_properties2_props.activeComputeUnitCount; - } else if (device->vendor_id == VK_VENDOR_ID_INTEL) { - device->shader_core_count = ggml_vk_intel_shader_core_count(device->physical_device); - } else { - device->shader_core_count = 0; - } - device->float_controls_rte_fp16 = vk12_props.shaderRoundingModeRTEFloat16; - device->float_controls_denorm_preserve_fp16 = vk12_props.shaderDenormPreserveFloat16; + VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE dot2_features {}; + dot2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE; + if (dot2_f16_support) { + last_struct->pNext = (VkBaseOutStructure *)&dot2_features; + last_struct = (VkBaseOutStructure *)&dot2_features; + } - device->subgroup_basic = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && - (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eBasic); - device->subgroup_arithmetic = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && - (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eArithmetic); -#ifdef __APPLE__ - // Workaround for subgroup arithmetic failing on MoltenVK with AMD GPUs (issue 15846) - if (device->vendor_id == VK_VENDOR_ID_AMD) { - device->subgroup_arithmetic = false; - } -#endif - device->subgroup_shuffle = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && - (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eShuffle); -#ifdef __APPLE__ - if (device->vendor_id == VK_VENDOR_ID_AMD) { - device->subgroup_shuffle = false; - } +#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) + VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT ocp_microscaling_features {}; + ocp_microscaling_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OCP_MICROSCALING_TYPES_FEATURES_EXT; + VkPhysicalDeviceShaderFloat8FeaturesEXT shader_float8_features {}; + shader_float8_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT; + if (ocp_microscaling_extension) { + last_struct->pNext = (VkBaseOutStructure *)&ocp_microscaling_features; + last_struct = (VkBaseOutStructure *)&ocp_microscaling_features; + } + if (shader_float8_extension) { + last_struct->pNext = (VkBaseOutStructure *)&shader_float8_features; + last_struct = (VkBaseOutStructure *)&shader_float8_features; + } #endif - device->subgroup_clustered = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && - (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eClustered); - - device->subgroup_ballot = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && - (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eBallot); - device->subgroup_vote = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && - (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eVote); + vkGetPhysicalDeviceFeatures2(physical_device, &device_features2); - // Submit at least every 100 nodes, in case there are workloads without as much matmul. - device->max_nodes_per_submit = 100; - const char* GGML_VK_MAX_NODES_PER_SUBMIT = getenv("GGML_VK_MAX_NODES_PER_SUBMIT"); - if (GGML_VK_MAX_NODES_PER_SUBMIT != nullptr) { - uint32_t max_nodes_per_submit = std::stoul(GGML_VK_MAX_NODES_PER_SUBMIT); - device->max_nodes_per_submit = std::max(max_nodes_per_submit, 1u); - } + fp16 = fp16 && vk12_features.shaderFloat16; - const bool force_disable_f16 = getenv("GGML_VK_DISABLE_F16") != nullptr; +#if defined(VK_KHR_shader_bfloat16) + bool bf16 = bfloat16_support && bfloat16_features.shaderBFloat16Type; +#else + bool bf16 = false; +#endif - device->fp16 = !force_disable_f16 && fp16_storage && fp16_compute; + uint32_t default_subgroup_size = get_subgroup_size("", device_architecture); + const size_t subgroup_size = (default_subgroup_size != 0) ? default_subgroup_size : subgroup_props.subgroupSize; + const bool uma = props2.properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; - if (!ggml_vk_khr_cooperative_matrix_support(device->properties, driver_props, device->architecture)) { - device->coopmat_support = false; - } + integer_dot_product = integer_dot_product + && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated + && shader_integer_dot_product_features.shaderIntegerDotProduct; - device->integer_dot_product = device->integer_dot_product && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated; + coopmat_support = coopmat_support +#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + && coopmat_features.cooperativeMatrix +#endif + && ggml_vk_khr_cooperative_matrix_support(props2.properties, driver_props, device_architecture); - device->min_imported_host_pointer_alignment = external_memory_host_props.minImportedHostPointerAlignment; +#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) + coopmat2_support = coopmat2_support && + coopmat2_features.cooperativeMatrixWorkgroupScope && + coopmat2_features.cooperativeMatrixFlexibleDimensions && + coopmat2_features.cooperativeMatrixReductions && + coopmat2_features.cooperativeMatrixConversions && + coopmat2_features.cooperativeMatrixPerElementOperations && + coopmat2_features.cooperativeMatrixTensorAddressing && + coopmat2_features.cooperativeMatrixBlockLoads; +#else + coopmat2_support = false; +#endif - device->max_workgroup_size_log2 = uint32_t(log2f(float(device->properties.limits.maxComputeWorkGroupInvocations))); + coopmat2_decode_vector_support = coopmat2_decode_vector_support && coopmat2_decode_vector_features.cooperativeMatrixDecodeVector; +#if !defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT) + coopmat2_decode_vector_support = false; +#endif - std::vector<vk::QueueFamilyProperties> queue_family_props = device->physical_device.getQueueFamilyProperties(); + std::string matrix_cores = coopmat2_support ? (coopmat2_decode_vector_support ? "NV_coopmat2v" : "NV_coopmat2") + : coopmat_support ? "KHR_coopmat" + : "none"; - // Try to find a non-graphics compute queue and transfer-focused queues - // Allow overriding avoiding the graphics queue because it can increase performance on RADV - const bool allow_graphics_queue = (getenv("GGML_VK_ALLOW_GRAPHICS_QUEUE") != nullptr); - const vk::QueueFlagBits graphics_flag = allow_graphics_queue ? (vk::QueueFlagBits)0 : vk::QueueFlagBits::eGraphics; - const uint32_t compute_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eCompute, graphics_flag, -1, 1); - const uint32_t transfer_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eTransfer, vk::QueueFlagBits::eCompute | graphics_flag, compute_queue_family_index, 1); + bool dot2_f16 = dot2_f16_support && dot2_features.shaderMixedFloatDotProductFloat16AccFloat32; + const char *fp16_str = fp16 ? (dot2_f16 ? "dot2" : "1") : "0"; +#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) + const bool fp4 = ocp_microscaling_extension && ocp_microscaling_features.shaderFloat4 && + shader_float8_extension && shader_float8_features.shaderFloat8 && + !getenv("GGML_VK_DISABLE_OCP_FP4"); +#else + GGML_UNUSED(ocp_microscaling_extension); + GGML_UNUSED(shader_float8_extension); + const bool fp4 = false; +#endif - const float priorities[] = { 1.0f, 1.0f }; - device->single_queue = compute_queue_family_index == transfer_queue_family_index && queue_family_props[compute_queue_family_index].queueCount == 1; + std::string device_name = props2.properties.deviceName.data(); + GGML_LOG_DEBUG("ggml_vulkan: %zu = %s (%s) | uma: %d | fp16: %s | bf16: %d | fp4: %d | warp size: %zu | shared memory: %d | int dot: %d | matrix cores: %s\n", + idx, device_name.c_str(), driver_props.driverName.data(), uma, fp16_str, bf16, fp4, subgroup_size, + props2.properties.limits.maxComputeSharedMemorySize, integer_dot_product, matrix_cores.c_str()); - std::vector<vk::DeviceQueueCreateInfo> device_queue_create_infos; - vk::DeviceCreateInfo device_create_info{}; - std::vector<const char *> device_extensions; - vk::PhysicalDeviceFeatures device_features = device->physical_device.getFeatures(); + if (props2.properties.deviceType == vk::PhysicalDeviceType::eCpu) { + GGML_LOG_DEBUG("ggml_vulkan: Warning: Device type is CPU. This is probably not the device you want.\n"); + } +} - VkPhysicalDeviceFeatures2 device_features2; - device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; - device_features2.pNext = nullptr; - device_features2.features = (VkPhysicalDeviceFeatures)device_features; +static DispatchLoaderDynamic ggml_vk_default_dispatcher_instance; - VkPhysicalDeviceVulkan11Features vk11_features; - vk11_features.pNext = nullptr; - vk11_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES; - device_features2.pNext = &vk11_features; +DispatchLoaderDynamic & ggml_vk_default_dispatcher() { + return ggml_vk_default_dispatcher_instance; +} - VkPhysicalDeviceVulkan12Features vk12_features; - vk12_features.pNext = nullptr; - vk12_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; - vk11_features.pNext = &vk12_features; +void ggml_vk_instance_init() { + if (vk_instance_initialized) { + return; + } + VK_LOG_DEBUG("ggml_vk_instance_init()"); - last_struct = (VkBaseOutStructure *)&vk12_features; + // See https://github.com/KhronosGroup/Vulkan-Hpp?tab=readme-ov-file#extensions--per-device-function-pointers- + ggml_vk_default_dispatcher_instance.init(vkGetInstanceProcAddr); - VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR internally_synchronized_queues_features{}; - internally_synchronized_queues_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INTERNALLY_SYNCHRONIZED_QUEUES_FEATURES_KHR; - internally_synchronized_queues_features.pNext = nullptr; - internally_synchronized_queues_features.internallySynchronizedQueues = VK_FALSE; + uint32_t api_version = vk::enumerateInstanceVersion(); - if (internally_sync_support) { - last_struct->pNext = (VkBaseOutStructure *)&internally_synchronized_queues_features; - last_struct = (VkBaseOutStructure *)&internally_synchronized_queues_features; - device_extensions.push_back(VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME); - } + if (api_version < VK_API_VERSION_1_2) { + std::cerr << "ggml_vulkan: Error: Vulkan 1.2 required." << std::endl; + throw vk::SystemError(vk::Result::eErrorFeatureNotPresent, "Vulkan 1.2 required"); + } - VkPhysicalDevicePipelineRobustnessFeaturesEXT pl_robustness_features; - pl_robustness_features.pNext = nullptr; - pl_robustness_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_ROBUSTNESS_FEATURES_EXT; - pl_robustness_features.pipelineRobustness = VK_FALSE; + vk::ApplicationInfo app_info{ "ggml-vulkan", 1, nullptr, 0, api_version }; - if (pipeline_robustness) { - last_struct->pNext = (VkBaseOutStructure *)&pl_robustness_features; - last_struct = (VkBaseOutStructure *)&pl_robustness_features; - device_extensions.push_back("VK_EXT_pipeline_robustness"); - } - - VkPhysicalDeviceMemoryPriorityFeaturesEXT memory_priority_features; - memory_priority_features.pNext = nullptr; - memory_priority_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PRIORITY_FEATURES_EXT; - memory_priority_features.memoryPriority = VK_FALSE; - if (device->memory_priority) { - last_struct->pNext = (VkBaseOutStructure *)&memory_priority_features; - last_struct = (VkBaseOutStructure *)&memory_priority_features; - device_extensions.push_back("VK_EXT_memory_priority"); - } - - VkPhysicalDeviceSubgroupSizeControlFeaturesEXT subgroup_size_control_features; - subgroup_size_control_features.pNext = nullptr; - subgroup_size_control_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT; - subgroup_size_control_features.computeFullSubgroups = false; - subgroup_size_control_features.subgroupSizeControl = false; - - if (device->subgroup_size_control) { - last_struct->pNext = (VkBaseOutStructure *)&subgroup_size_control_features; - last_struct = (VkBaseOutStructure *)&subgroup_size_control_features; - } - -#if defined(VK_KHR_cooperative_matrix) - VkPhysicalDeviceCooperativeMatrixFeaturesKHR coopmat_features; - coopmat_features.pNext = nullptr; - coopmat_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR; - coopmat_features.cooperativeMatrix = VK_FALSE; - - if (device->coopmat_support) { - last_struct->pNext = (VkBaseOutStructure *)&coopmat_features; - last_struct = (VkBaseOutStructure *)&coopmat_features; - } + const std::vector<vk::ExtensionProperties> instance_extensions = vk::enumerateInstanceExtensionProperties(); + const bool layer_settings = ggml_vk_instance_layer_settings_available(); +#ifdef __APPLE__ + const bool portability_enumeration_ext = ggml_vk_instance_portability_enumeration_ext_available(instance_extensions); #endif + const bool debug_utils_ext = ggml_vk_instance_debug_utils_ext_available(instance_extensions) && getenv("GGML_VK_DEBUG_MARKERS") != nullptr; + std::vector<const char*> layers; -#if defined(VK_NV_cooperative_matrix2) - VkPhysicalDeviceCooperativeMatrix2FeaturesNV coopmat2_features {}; - coopmat2_features.pNext = nullptr; - coopmat2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_2_FEATURES_NV; - if (coopmat2_support) { - last_struct->pNext = (VkBaseOutStructure *)&coopmat2_features; - last_struct = (VkBaseOutStructure *)&coopmat2_features; - device_extensions.push_back("VK_NV_cooperative_matrix2"); - } + if (layer_settings) { + layers.push_back("VK_LAYER_KHRONOS_validation"); + } + std::vector<const char*> extensions; + if (layer_settings) { + extensions.push_back("VK_EXT_layer_settings"); + } +#ifdef __APPLE__ + if (portability_enumeration_ext) { + extensions.push_back("VK_KHR_portability_enumeration"); + } #endif - - VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV coopmat2_decode_vector_features {}; - coopmat2_decode_vector_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV; - if (coopmat2_decode_vector_support) { - last_struct->pNext = (VkBaseOutStructure *)&coopmat2_decode_vector_features; - last_struct = (VkBaseOutStructure *)&coopmat2_decode_vector_features; - device_extensions.push_back(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME); - } - -#if defined(VK_KHR_shader_bfloat16) - VkPhysicalDeviceShaderBfloat16FeaturesKHR bfloat16_features {}; - bfloat16_features.pNext = nullptr; - bfloat16_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR; - if (bfloat16_support) { - last_struct->pNext = (VkBaseOutStructure *)&bfloat16_features; - last_struct = (VkBaseOutStructure *)&bfloat16_features; - device_extensions.push_back("VK_KHR_shader_bfloat16"); - } + if (debug_utils_ext) { + extensions.push_back("VK_EXT_debug_utils"); + } + VkBool32 enable_best_practice = layer_settings; + std::vector<vk::LayerSettingEXT> settings = { + { + "VK_LAYER_KHRONOS_validation", + "validate_best_practices", + vk::LayerSettingTypeEXT::eBool32, + 1, + &enable_best_practice + }, + }; + vk::LayerSettingsCreateInfoEXT layer_setting_info(settings); + vk::InstanceCreateInfo instance_create_info(vk::InstanceCreateFlags{}, &app_info, layers, extensions, &layer_setting_info); +#ifdef __APPLE__ + if (portability_enumeration_ext) { + instance_create_info.flags |= vk::InstanceCreateFlagBits::eEnumeratePortabilityKHR; + } #endif - VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT ocp_microscaling_features {}; - ocp_microscaling_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OCP_MICROSCALING_TYPES_FEATURES_EXT; - if (ocp_microscaling_extension) { - last_struct->pNext = (VkBaseOutStructure *)&ocp_microscaling_features; - last_struct = (VkBaseOutStructure *)&ocp_microscaling_features; - device_extensions.push_back(VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME); - } + vk_instance.instance = vk::createInstance(instance_create_info); + vk_instance_initialized = true; - VkPhysicalDeviceShaderFloat8FeaturesEXT shader_float8_features {}; - shader_float8_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT; - if (shader_float8_extension) { - last_struct->pNext = (VkBaseOutStructure *)&shader_float8_features; - last_struct = (VkBaseOutStructure *)&shader_float8_features; - device_extensions.push_back(VK_EXT_SHADER_FLOAT8_EXTENSION_NAME); - } + if (debug_utils_ext) { + vk_instance.debug_utils_support = true; + vk_instance.pfn_vkSetDebugUtilsObjectNameEXT = (PFN_vkSetDebugUtilsObjectNameEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkSetDebugUtilsObjectNameEXT"); + vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT = (PFN_vkQueueBeginDebugUtilsLabelEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkQueueBeginDebugUtilsLabelEXT"); + vk_instance.pfn_vkQueueEndDebugUtilsLabelEXT = (PFN_vkQueueEndDebugUtilsLabelEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkQueueEndDebugUtilsLabelEXT"); + vk_instance.pfn_vkCmdBeginDebugUtilsLabelEXT = (PFN_vkCmdBeginDebugUtilsLabelEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkCmdBeginDebugUtilsLabelEXT"); + vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT = (PFN_vkCmdEndDebugUtilsLabelEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkCmdEndDebugUtilsLabelEXT"); + vk_instance.pfn_vkCmdInsertDebugUtilsLabelEXT = (PFN_vkCmdInsertDebugUtilsLabelEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkCmdInsertDebugUtilsLabelEXT"); + } - VkPhysicalDeviceMaintenance4Features maint4_features {}; - maint4_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_4_FEATURES; - if (maintenance4_support) { - last_struct->pNext = (VkBaseOutStructure *)&maint4_features; - last_struct = (VkBaseOutStructure *)&maint4_features; - device_extensions.push_back("VK_KHR_maintenance4"); + vk_perf_logger_enabled = getenv("GGML_VK_PERF_LOGGER") != nullptr; + vk_perf_logger_concurrent = getenv("GGML_VK_PERF_LOGGER_CONCURRENT") != nullptr; + { + const char * e = getenv("WP_VK_NODE_TS"); + vk_node_ts_enabled = e != nullptr && e[0] == '1'; + if (vk_node_ts_enabled) { + GGML_LOG_INFO("ggml_vulkan: WP_VK_NODE_TS=1 timestamps harvested at synchronize\n"); } + } + vk_enable_sync_logger = getenv("GGML_VK_SYNC_LOGGER") != nullptr; + vk_memory_logger_enabled = getenv("GGML_VK_MEMORY_LOGGER") != nullptr; + const char* GGML_VK_PIPELINE_STATS = getenv("GGML_VK_PIPELINE_STATS"); + if (GGML_VK_PIPELINE_STATS != nullptr) { + vk_pipeline_stats_filter = GGML_VK_PIPELINE_STATS; + } + const char* GGML_VK_PERF_LOGGER_FREQUENCY = getenv("GGML_VK_PERF_LOGGER_FREQUENCY"); - VkPhysicalDeviceShaderIntegerDotProductFeaturesKHR shader_integer_dot_product_features {}; - shader_integer_dot_product_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_INTEGER_DOT_PRODUCT_FEATURES_KHR; - if (device->integer_dot_product) { - last_struct->pNext = (VkBaseOutStructure *)&shader_integer_dot_product_features; - last_struct = (VkBaseOutStructure *)&shader_integer_dot_product_features; - device_extensions.push_back("VK_KHR_shader_integer_dot_product"); - } + if (GGML_VK_PERF_LOGGER_FREQUENCY != nullptr) { + vk_perf_logger_frequency = std::stoul(GGML_VK_PERF_LOGGER_FREQUENCY); + } - VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE dot2_features {}; - dot2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE; - if (dot2_f16_support) { - last_struct->pNext = (VkBaseOutStructure *)&dot2_features; - last_struct = (VkBaseOutStructure *)&dot2_features; - device_extensions.push_back("VK_VALVE_shader_mixed_float_dot_product"); - } + // See https://github.com/KhronosGroup/Vulkan-Hpp?tab=readme-ov-file#extensions--per-device-function-pointers- + VULKAN_HPP_DEFAULT_DISPATCHER.init(vk_instance.instance); - VkPhysicalDevicePipelineExecutablePropertiesFeaturesKHR pep_features {}; - pep_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_EXECUTABLE_PROPERTIES_FEATURES_KHR; - if (pipeline_executable_properties_support) { - last_struct->pNext = (VkBaseOutStructure *)&pep_features; - last_struct = (VkBaseOutStructure *)&pep_features; - device_extensions.push_back("VK_KHR_pipeline_executable_properties"); - } + std::vector<vk::PhysicalDevice> devices = vk_instance.instance.enumeratePhysicalDevices(); - if (device->external_memory_host) { - device_extensions.push_back("VK_EXT_external_memory_host"); - } + // Emulate behavior of CUDA_VISIBLE_DEVICES for Vulkan + char * devices_env = getenv("GGML_VK_VISIBLE_DEVICES"); + if (devices_env != nullptr) { + size_t num_available_devices = devices.size(); - if (device->external_semaphore_fd) { - device_extensions.push_back("VK_KHR_external_semaphore_fd"); - } + std::string devices(devices_env); + std::replace(devices.begin(), devices.end(), ',', ' '); -#if defined(VK_EXT_shader_64bit_indexing) - VkPhysicalDeviceShader64BitIndexingFeaturesEXT shader_64bit_indexing_features {}; - shader_64bit_indexing_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_64_BIT_INDEXING_FEATURES_EXT; - if (device->shader_64b_indexing) { - last_struct->pNext = (VkBaseOutStructure *)&shader_64bit_indexing_features; - last_struct = (VkBaseOutStructure *)&shader_64bit_indexing_features; - device_extensions.push_back("VK_EXT_shader_64bit_indexing"); + std::stringstream ss(devices); + size_t tmp; + while (ss >> tmp) { + if(tmp >= num_available_devices) { + std::cerr << "ggml_vulkan: Invalid device index " << tmp << " in GGML_VK_VISIBLE_DEVICES." << std::endl; + throw std::runtime_error("Invalid Vulkan device index"); + } + vk_instance.device_indices.push_back(tmp); } -#endif - - VkPhysicalDeviceFaultFeaturesEXT fault_features {}; - fault_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FAULT_FEATURES_EXT; - if (device->device_fault) { - last_struct->pNext = (VkBaseOutStructure *)&fault_features; - last_struct = (VkBaseOutStructure *)&fault_features; - device_extensions.push_back("VK_EXT_device_fault"); + } else { + // If no vulkan devices are found, return early + if (devices.empty()) { + GGML_LOG_INFO("ggml_vulkan: No devices found.\n"); + return; } - vkGetPhysicalDeviceFeatures2(device->physical_device, &device_features2); - - device->device_fault = device->device_fault && fault_features.deviceFault; + // Default to using all dedicated GPUs + for (size_t i = 0; i < devices.size(); i++) { + vk::PhysicalDeviceProperties2 new_props; + vk::PhysicalDeviceDriverProperties new_driver; + vk::PhysicalDeviceIDProperties new_id; + new_props.pNext = &new_driver; + new_driver.pNext = &new_id; + devices[i].getProperties2(&new_props); - device->has_internally_synchronized_queues = internally_synchronized_queues_features.internallySynchronizedQueues; + if ((new_props.properties.deviceType == vk::PhysicalDeviceType::eDiscreteGpu || new_props.properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu) && ggml_vk_device_is_supported(devices[i])) { + // Check if there are two physical devices corresponding to the same GPU + // This handles the case where the same GPU appears with different drivers (e.g., RADV + AMDVLK on Linux), + // see https://github.com/ggml-org/llama.cpp/pull/7582 for original deduplication. + // MoltenVK on macOS may report the same UUID for distinct GPUs on multi-GPU cards, + // see https://github.com/KhronosGroup/MoltenVK/issues/2683. Skip when both old/new + // driver is MoltenVK + auto old_device = std::find_if( + vk_instance.device_indices.begin(), + vk_instance.device_indices.end(), + [&devices, &new_id, &new_driver](const size_t k){ + vk::PhysicalDeviceProperties2 old_props; + vk::PhysicalDeviceDriverProperties old_driver; + vk::PhysicalDeviceIDProperties old_id; + old_props.pNext = &old_driver; + old_driver.pNext = &old_id; + devices[k].getProperties2(&old_props); - // Build queue create infos only after querying whether internally synchronized queues are enabled. - // getQueue2() later uses the same flag, so creation/retrieval must stay consistent. - vk::DeviceQueueCreateFlags queue_flags = device->has_internally_synchronized_queues ? - eInternallySynchronizedKHR : - vk::DeviceQueueCreateFlags(); + bool same_uuid = std::equal(std::begin(old_id.deviceUUID), std::end(old_id.deviceUUID), std::begin(new_id.deviceUUID)); + same_uuid = same_uuid || ( + old_id.deviceLUIDValid && new_id.deviceLUIDValid && + std::equal(std::begin(old_id.deviceLUID), std::end(old_id.deviceLUID), std::begin(new_id.deviceLUID)) + ); + bool both_molten_vk = (new_driver.driverID == vk::DriverId::eMoltenvk && old_driver.driverID == vk::DriverId::eMoltenvk); - if (compute_queue_family_index != transfer_queue_family_index) { - device_queue_create_infos.push_back({queue_flags, compute_queue_family_index, 1, priorities}); - device_queue_create_infos.push_back({queue_flags, transfer_queue_family_index, 1, priorities + 1}); - } else if(!device->single_queue) { - device_queue_create_infos.push_back({queue_flags, compute_queue_family_index, 2, priorities}); - } else { - device_queue_create_infos.push_back({queue_flags, compute_queue_family_index, 1, priorities}); - } + return same_uuid && !both_molten_vk; + } + ); + if (old_device == vk_instance.device_indices.end()) { + vk_instance.device_indices.push_back(i); + } else { + // There can be two physical devices corresponding to the same GPU if there are 2 different drivers + // This can cause error when splitting layers aross the devices, need to keep only 1 + VK_LOG_DEBUG("Device " << i << " and device " << *old_device << " have the same deviceUUID"); - device->pipeline_executable_properties_support = pipeline_executable_properties_support; + vk::PhysicalDeviceProperties2 old_props; + vk::PhysicalDeviceDriverProperties old_driver; + old_props.pNext = &old_driver; + devices[*old_device].getProperties2(&old_props); - device->fp16 = device->fp16 && vk12_features.shaderFloat16; + std::map<vk::DriverId, int> driver_priorities {}; + int old_priority = std::numeric_limits<int>::max(); + int new_priority = std::numeric_limits<int>::max(); -#if defined(VK_KHR_shader_bfloat16) - device->bf16 = bfloat16_support && bfloat16_features.shaderBFloat16Type; -#else - device->bf16 = false; + // Check https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkDriverId.html for the list of driver id + // Smaller number -> higher priority + switch (old_props.properties.vendorID) { + case VK_VENDOR_ID_AMD: + driver_priorities[vk::DriverId::eMesaRadv] = 1; + driver_priorities[vk::DriverId::eAmdOpenSource] = 2; + driver_priorities[vk::DriverId::eAmdProprietary] = 3; + break; + case VK_VENDOR_ID_INTEL: + driver_priorities[vk::DriverId::eIntelOpenSourceMESA] = 1; + driver_priorities[vk::DriverId::eIntelProprietaryWindows] = 2; + break; + case VK_VENDOR_ID_NVIDIA: + driver_priorities[vk::DriverId::eNvidiaProprietary] = 1; +#if defined(VK_API_VERSION_1_3) && VK_HEADER_VERSION >= 235 + driver_priorities[vk::DriverId::eMesaNvk] = 2; #endif + break; + case VK_VENDOR_ID_QUALCOMM: + driver_priorities[vk::DriverId::eQualcommProprietary] = 1; + driver_priorities[vk::DriverId::eMesaTurnip] = 2; + break; + } + driver_priorities[vk::DriverId::eMesaDozen] = 100; - device->dot2_f16 = dot2_f16_support && dot2_features.shaderMixedFloatDotProductFloat16AccFloat32; - device->ocp_fp4 = ocp_microscaling_extension && ocp_microscaling_features.shaderFloat4 && - shader_float8_extension && shader_float8_features.shaderFloat8 && - !getenv("GGML_VK_DISABLE_OCP_FP4"); + if (driver_priorities.count(old_driver.driverID)) { + old_priority = driver_priorities[old_driver.driverID]; + } + if (driver_priorities.count(new_driver.driverID)) { + new_priority = driver_priorities[new_driver.driverID]; + } - device->pipeline_robustness = pl_robustness_features.pipelineRobustness; + if (new_priority < old_priority) { + auto r = std::remove(vk_instance.device_indices.begin(), vk_instance.device_indices.end(), *old_device); + vk_instance.device_indices.erase(r, vk_instance.device_indices.end()); + vk_instance.device_indices.push_back(i); - // f32 multi_add needs 16-bit storage, not fp16 RTE. Default = RTE gate. - // GGML_VK_MULTI_ADD_F32_NO_RTE=1 creates f32 pipelines without RTE. - const bool multi_add_common = - device->properties.limits.maxPushConstantsSize >= sizeof(vk_op_multi_add_push_constants) && - getenv("GGML_VK_DISABLE_MULTI_ADD") == nullptr; - const char * f32_no_rte_env = getenv("GGML_VK_MULTI_ADD_F32_NO_RTE"); - const bool f32_no_rte = - f32_no_rte_env != nullptr && f32_no_rte_env[0] == '1' && - vk11_features.storageBuffer16BitAccess; - device->multi_add = multi_add_common && - (device->float_controls_rte_fp16 || f32_no_rte); + VK_LOG_DEBUG("Prioritize device " << i << " driver " << new_driver.driverName << " over device " << *old_device << " driver " << old_driver.driverName); + } + else { + VK_LOG_DEBUG("Prioritize device " << *old_device << " driver " << old_driver.driverName << " over device " << i << " driver " << new_driver.driverName << std::endl); + } + } + } + } - device->shader_int64 = device_features2.features.shaderInt64; - device->buffer_device_address = vk12_features.bufferDeviceAddress; - device->vulkan_memory_model = vk12_features.vulkanMemoryModel; + // If no GPUs found, fall back to the first non-CPU device. + // If only CPU devices are available, return without devices. + if (vk_instance.device_indices.empty()) { + for (size_t i = 0; i < devices.size(); i++) { + if (devices[i].getProperties().deviceType != vk::PhysicalDeviceType::eCpu) { + vk_instance.device_indices.push_back(i); + break; + } + } + } - if (device->subgroup_size_control) { - device->subgroup_min_size = subgroup_size_control_props.minSubgroupSize; - device->subgroup_max_size = subgroup_size_control_props.maxSubgroupSize; - device_extensions.push_back("VK_EXT_subgroup_size_control"); + if (vk_instance.device_indices.empty()) { + GGML_LOG_INFO("ggml_vulkan: No devices found.\n"); + return; } + } + GGML_LOG_DEBUG("ggml_vulkan: Found %zu Vulkan devices:\n", vk_instance.device_indices.size()); - device->subgroup_size_control = device->subgroup_size_control && - (subgroup_size_control_props.requiredSubgroupSizeStages & vk::ShaderStageFlagBits::eCompute) && - subgroup_size_control_features.subgroupSizeControl; + for (size_t i = 0; i < vk_instance.device_indices.size(); i++) { + vk::PhysicalDevice vkdev = devices[vk_instance.device_indices[i]]; + std::vector<vk::ExtensionProperties> extensionprops = vkdev.enumerateDeviceExtensionProperties(); - device->subgroup_require_full_support = subgroup_size_control_features.computeFullSubgroups; + bool membudget_supported = false; + for (const auto & ext : extensionprops) { + if (strcmp(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME, ext.extensionName) == 0) { + membudget_supported = true; + break; + } + } -#if defined(VK_KHR_cooperative_matrix) - device->coopmat_support = device->coopmat_support && coopmat_features.cooperativeMatrix; - device->coopmat1_fa_support = device->coopmat_support && device->subgroup_require_full_support; -#endif + vk_instance.device_supports_membudget.push_back(membudget_supported); - if (coopmat2_support) { -#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) - if (coopmat2_features.cooperativeMatrixWorkgroupScope && - coopmat2_features.cooperativeMatrixFlexibleDimensions && - coopmat2_features.cooperativeMatrixReductions && - coopmat2_features.cooperativeMatrixConversions && - coopmat2_features.cooperativeMatrixPerElementOperations && - coopmat2_features.cooperativeMatrixTensorAddressing && - coopmat2_features.cooperativeMatrixBlockLoads && - vk12_features.bufferDeviceAddress) { + ggml_vk_print_gpu_info(i); + } +} - std::vector<VkCooperativeMatrixFlexibleDimensionsPropertiesNV> flexible_dimensions; - uint32_t count = 0; +void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { + VK_LOG_DEBUG("ggml_vk_init(" << ctx->name << ", " << idx << ")"); + ggml_vk_instance_init(); + GGML_ASSERT(idx < vk_instance.device_indices.size()); - PFN_vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV - _vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV = - (PFN_vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV) - vk_instance.instance.getProcAddr("vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV"); + ctx->name = GGML_VK_NAME + std::to_string(idx); - _vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV(device->physical_device, &count, nullptr); + ctx->device = ggml_vk_get_device(idx); - VkCooperativeMatrixFlexibleDimensionsPropertiesNV empty_prop {}; - empty_prop.sType = VK_STRUCTURE_TYPE_COOPERATIVE_MATRIX_FLEXIBLE_DIMENSIONS_PROPERTIES_NV; - flexible_dimensions.resize(count, empty_prop); + ctx->semaphore_idx = 0; + ctx->event_idx = 0; - _vkGetPhysicalDeviceCooperativeMatrixFlexibleDimensionsPropertiesNV(device->physical_device, &count, flexible_dimensions.data()); + ctx->prealloc_size_x = 0; + ctx->prealloc_size_y = 0; + ctx->prealloc_size_split_k = 0; + // Fixed size of 1KB, for deterministic behavior + ctx->prealloc_size_add_rms_partials = 1024; - bool found_fp16_128 = false, - found_fp16_256 = false, - found_fp32_128 = false, - found_fp32_256 = false; - bool found_bf16_128 = false, - found_bf16_256 = false; - // need to support fp16*fp16 with fp16/fp32 accumulator, for workgroupsize 128 - // with 32x16x16 and 256 with 32x32x16. - for (auto &prop : flexible_dimensions) { - if (prop.saturatingAccumulation == VK_FALSE && - prop.scope == VK_SCOPE_WORKGROUP_KHR) { + ctx->fence = ctx->device->device.createFence({}); + ctx->almost_ready_fence = ctx->device->device.createFence({}); - if (prop.AType == VK_COMPONENT_TYPE_FLOAT16_KHR && - prop.BType == VK_COMPONENT_TYPE_FLOAT16_KHR) { + ctx->compute_cmd_pool.init(ctx->device, ctx->device->compute_queue.get()); + if (ctx->device->async_use_transfer_queue) { + vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eTimeline, 0 }; + vk::SemaphoreCreateInfo ci{}; + ci.setPNext(&tci); + ctx->transfer_semaphore.s = ctx->device->device.createSemaphore(ci); + ctx->transfer_semaphore.value = 0; - if (prop.workgroupInvocations == 128 && - prop.MGranularity <= 32 && - prop.NGranularity <= 16 && - prop.KGranularity <= 16) { - if (prop.CType == VK_COMPONENT_TYPE_FLOAT16_KHR && - prop.ResultType == VK_COMPONENT_TYPE_FLOAT16_KHR) { - found_fp16_128 = true; - } - if (prop.CType == VK_COMPONENT_TYPE_FLOAT32_KHR && - prop.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR) { - found_fp32_128 = true; - } - } - if (prop.workgroupInvocations == 256 && - prop.MGranularity <= 32 && - prop.NGranularity <= 32 && - prop.KGranularity <= 16) { - if (prop.CType == VK_COMPONENT_TYPE_FLOAT16_KHR && - prop.ResultType == VK_COMPONENT_TYPE_FLOAT16_KHR) { - found_fp16_256 = true; - } - if (prop.CType == VK_COMPONENT_TYPE_FLOAT32_KHR && - prop.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR) { - found_fp32_256 = true; - } - } - } + ctx->transfer_cmd_pool.init(ctx->device, ctx->device->transfer_queue.get()); + } -#if defined(VK_KHR_shader_bfloat16) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - if (bfloat16_support && - prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR && - prop.BType == VK_COMPONENT_TYPE_BFLOAT16_KHR && - prop.CType == VK_COMPONENT_TYPE_FLOAT32_KHR && - prop.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR) { + if (vk_perf_logger_enabled) { + ctx->perf_logger = std::unique_ptr<vk_perf_logger>(new vk_perf_logger()); + } - if (prop.workgroupInvocations == 128 && - prop.MGranularity <= 32 && - prop.NGranularity <= 16 && - prop.KGranularity <= 16) { - found_bf16_128 = true; - } - if (prop.workgroupInvocations == 256 && - prop.MGranularity <= 32 && - prop.NGranularity <= 32 && - prop.KGranularity <= 16) { - found_bf16_256 = true; - } - } -#endif - } - } - if (found_fp16_128 && found_fp16_256 && - found_fp32_128 && found_fp32_256 && - coopmat2_props.cooperativeMatrixFlexibleDimensionsMaxDimension >= 512) { - device->coopmat2 = true; - device->coopmat2_bf16_support = found_bf16_128 && found_bf16_256; - device->coopmat2_decode_vector = coopmat2_decode_vector_support && coopmat2_decode_vector_features.cooperativeMatrixDecodeVector; - } - } +#ifdef GGML_VULKAN_CHECK_RESULTS + const char* skip_checks = getenv("GGML_VULKAN_SKIP_CHECKS"); + vk_skip_checks = (skip_checks == NULL ? 0 : atoi(skip_checks)); + const char* output_tensor = getenv("GGML_VULKAN_OUTPUT_TENSOR"); + vk_output_tensor = (output_tensor == NULL ? 0 : atoi(output_tensor)); #endif - } - - if (!vk11_features.storageBuffer16BitAccess) { - std::cerr << "ggml_vulkan: device " << GGML_VK_NAME << idx << " does not support 16-bit storage." << std::endl; - throw std::runtime_error("Unsupported device"); - } +} - device_extensions.push_back("VK_KHR_16bit_storage"); - -#ifdef GGML_VULKAN_VALIDATE - device_extensions.push_back("VK_KHR_shader_non_semantic_info"); -#endif - - if (device->fp16) { - device_extensions.push_back("VK_KHR_shader_float16_int8"); - } - -#if defined(VK_KHR_cooperative_matrix) - if (device->coopmat_support) { - // Query supported shapes - std::vector<VkCooperativeMatrixPropertiesKHR> cm_props; - - PFN_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR pfn_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR = - (PFN_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR)vkGetInstanceProcAddr(vk_instance.instance, "vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR"); - - uint32_t cm_props_num; - - pfn_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR(device->physical_device, &cm_props_num, nullptr); - - cm_props.resize(cm_props_num); +vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type) { + VK_LOG_DEBUG("ggml_vk_get_to_fp16()"); + switch (type) { + case GGML_TYPE_F32: + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_TQ1_0: + case GGML_TYPE_TQ2_0: + break; + default: + return nullptr; + } - for (auto& prop : cm_props) { - prop.sType = VK_STRUCTURE_TYPE_COOPERATIVE_MATRIX_PROPERTIES_KHR; - } + return ctx->device->pipeline_dequant[type]; +} - pfn_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR(device->physical_device, &cm_props_num, cm_props.data()); +static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * ctx, ggml_type a_type, ggml_type b_type, uint32_t num_cols, uint32_t m, uint32_t k) { + VK_LOG_DEBUG("ggml_vk_get_dequantize_mul_mat_vec()"); + GGML_ASSERT(b_type == GGML_TYPE_F32 || b_type == GGML_TYPE_F16 || b_type == GGML_TYPE_Q8_1); + GGML_ASSERT(num_cols >= 1 && num_cols <= ctx->device->mul_mat_vec_max_cols_eff); - VK_LOG_DEBUG("ggml_vulkan: Cooperative Matrix Shapes: " << cm_props.size()); + if (b_type == GGML_TYPE_Q8_1) { + switch (a_type) { + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_XS: + break; + default: + return nullptr; + } + } - for (auto& prop : cm_props) { - VK_LOG_DEBUG("ggml_vulkan: M: " << prop.MSize << " N: " << prop.NSize << " K: " << prop.KSize << " A: " << vk::to_string((vk::ComponentTypeKHR)prop.AType) << " B: " << vk::to_string((vk::ComponentTypeKHR)prop.BType) << " C: " << vk::to_string((vk::ComponentTypeKHR)prop.CType) << " Result: " << vk::to_string((vk::ComponentTypeKHR)prop.ResultType) << " saturatingAccumulation: " << prop.saturatingAccumulation << " scope: " << vk::to_string((vk::ScopeKHR)prop.scope)); + switch (a_type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_TQ1_0: + case GGML_TYPE_TQ2_0: + break; + default: + return nullptr; + } - if ((vk::ComponentTypeKHR)prop.AType == vk::ComponentTypeKHR::eFloat16 && - (vk::ComponentTypeKHR)prop.BType == vk::ComponentTypeKHR::eFloat16 && - (vk::ScopeKHR)prop.scope == vk::ScopeKHR::eSubgroup - ) { - if ((vk::ComponentTypeKHR)prop.CType == vk::ComponentTypeKHR::eFloat32 && - (vk::ComponentTypeKHR)prop.ResultType == vk::ComponentTypeKHR::eFloat32) { - // coopmat sizes not set yet - if (device->coopmat_m == 0) { - device->coopmat_acc_f32_support = true; - device->coopmat_m = prop.MSize; - device->coopmat_n = prop.NSize; - device->coopmat_k = prop.KSize; - } else if (device->coopmat_m == prop.MSize && device->coopmat_n == prop.NSize && device->coopmat_k == prop.KSize) { - // Only enable if shape is identical - device->coopmat_acc_f32_support = true; - } - if (prop.MSize == 16 && prop.NSize == 16 && prop.KSize == 16) { - device->coopmat_support_16x16x16_f32acc = true; - } - } else if ((vk::ComponentTypeKHR)prop.CType == vk::ComponentTypeKHR::eFloat16 && - (vk::ComponentTypeKHR)prop.ResultType == vk::ComponentTypeKHR::eFloat16) { - // coopmat sizes not set yet - if (device->coopmat_m == 0) { - device->coopmat_acc_f16_support = true; - device->coopmat_m = prop.MSize; - device->coopmat_n = prop.NSize; - device->coopmat_k = prop.KSize; - } else if (device->coopmat_m == prop.MSize && device->coopmat_n == prop.NSize && device->coopmat_k == prop.KSize) { - // Only enable if shape is identical - device->coopmat_acc_f16_support = true; - } - if (prop.MSize == 16 && prop.NSize == 16 && prop.KSize == 16) { - device->coopmat_support_16x16x16_f16acc = true; - } - } - } else if ((vk::ComponentTypeKHR)prop.AType == vk::ComponentTypeKHR::eSint8 && - (vk::ComponentTypeKHR)prop.BType == vk::ComponentTypeKHR::eSint8 && - (vk::ComponentTypeKHR)prop.CType == vk::ComponentTypeKHR::eSint32 && - (vk::ComponentTypeKHR)prop.ResultType == vk::ComponentTypeKHR::eSint32 && - (vk::ScopeKHR)prop.scope == vk::ScopeKHR::eSubgroup && - device->coopmat_int_m == 0 - ) { - device->coopmat_int_support = true; - device->coopmat_int_m = prop.MSize; - device->coopmat_int_n = prop.NSize; - device->coopmat_int_k = prop.KSize; - } -#if defined(VK_KHR_shader_bfloat16) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - if (bfloat16_support && - prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR && - prop.BType == VK_COMPONENT_TYPE_BFLOAT16_KHR && - prop.CType == VK_COMPONENT_TYPE_FLOAT32_KHR && - prop.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR && - (vk::ScopeKHR)prop.scope == vk::ScopeKHR::eSubgroup - ) { - // coopmat sizes not set yet - if (device->coopmat_m == 0) { - device->coopmat_bf16_support = true; - device->coopmat_m = prop.MSize; - device->coopmat_n = prop.NSize; - device->coopmat_k = prop.KSize; - } else if (device->coopmat_m == prop.MSize && device->coopmat_n == prop.NSize && device->coopmat_k == prop.KSize) { - // Only enable if shape is identical - device->coopmat_bf16_support = true; - } - } -#endif + // heuristic to choose workgroup size + uint32_t dmmv_wg = DMMV_WG_SIZE_SUBGROUP; + if ((ctx->device->vendor_id == VK_VENDOR_ID_NVIDIA && ctx->device->architecture != vk_device_architecture::NVIDIA_PRE_TURING) || ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { + // NOTE (2026-08-17): tried adding AMD_GCN here (DMMV_WG_SIZE_LARGE for the + // MoE expert mat-vec) -- MEASURED a REGRESSION on gfx803/RX480: 9.74 -> + // 13.63 ms/req. The larger workgroup's barrier/tree-reduction overhead + // outweighs the extra parallelism on Polaris. GCN stays on SUBGROUP. + // Prefer larger workgroups when M is small, to spread the work out more + // and keep more SMs busy. + // q6_k seems to prefer small workgroup size even for "medium" values of M. + if (a_type == GGML_TYPE_Q6_K) { + if (m < 4096 && k >= 1024) { + dmmv_wg = DMMV_WG_SIZE_LARGE; } - - if (device->coopmat_m == 0 || !device->coopmat_acc_f32_support) { - // No suitable matmul mode found - GGML_LOG_DEBUG("ggml_vulkan: WARNING: No suitable matrix core mode found. Disabling matrix cores.\n"); - device->coopmat_support = false; + } else { + if (m <= 8192 && k >= 1024) { + dmmv_wg = DMMV_WG_SIZE_LARGE; } } + } - if (device->coopmat_support) { - device_extensions.push_back("VK_KHR_cooperative_matrix"); + if (b_type == GGML_TYPE_Q8_1) { + if (ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { + dmmv_wg = DMMV_WG_SIZE_SUBGROUP; } -#endif - device->name = GGML_VK_NAME + std::to_string(idx); + return ctx->device->pipeline_dequant_mul_mat_vec_q8_1_f32[dmmv_wg][a_type][num_cols-1]; + } - device_create_info - .setFlags(vk::DeviceCreateFlags()) - .setQueueCreateInfos(device_queue_create_infos) - .setPEnabledExtensionNames(device_extensions); - device_create_info.setPNext(&device_features2); - device->device = device->physical_device.createDevice(device_create_info); + return b_type == GGML_TYPE_F32 ? ctx->device->pipeline_dequant_mul_mat_vec_f32_f32[dmmv_wg][a_type][num_cols-1] : ctx->device->pipeline_dequant_mul_mat_vec_f16_f32[dmmv_wg][a_type][num_cols-1]; +} - if (device->device_fault) { - device->pfn_vkGetDeviceFaultInfoEXT = (PFN_vkGetDeviceFaultInfoEXT) - vkGetDeviceProcAddr(device->device, "vkGetDeviceFaultInfoEXT"); - } +static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_context * ctx, ggml_type a_type, ggml_type b_type, uint32_t m, uint32_t k) { + VK_LOG_DEBUG("ggml_vk_get_dequantize_mul_mat_vec_id()"); + GGML_ASSERT(b_type == GGML_TYPE_F32 || b_type == GGML_TYPE_Q8_1); - // Queues - device->compute_queue = ggml_vk_create_queue(device, compute_queue_family_index, 0, { vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer }, false); + if (b_type == GGML_TYPE_Q8_1) { + switch (a_type) { + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_XS: + break; + default: + return nullptr; + } + } - // Shaders - // Disable matmul tile sizes early if performance low or not supported - for (uint32_t i = 0; i < GGML_TYPE_COUNT; ++i) { - switch (device->vendor_id) { -#ifndef GGML_VULKAN_RUN_TESTS - case VK_VENDOR_ID_AMD: - device->mul_mat_l[i] = device->coopmat_support && device->driver_id != vk::DriverId::eAmdProprietary; - device->mul_mat_m[i] = true; - device->mul_mat_s[i] = true; - device->mul_mat_id_l[i] = false; - device->mul_mat_id_m[i] = true; - device->mul_mat_id_s[i] = true; - break; - case VK_VENDOR_ID_INTEL: { - // Current Windows driver does not expose BF16 support. - // We only want to use l_warptile if coopmat is available - const bool use_l_warptile = (i == GGML_TYPE_BF16) ? (device->coopmat_bf16_support && device->coopmat_support) : device->coopmat_support; - device->mul_mat_l[i] = use_l_warptile; - device->mul_mat_id_l[i] = use_l_warptile; - device->mul_mat_m[i] = true; - device->mul_mat_s[i] = true; - device->mul_mat_id_m[i] = true; - device->mul_mat_id_s[i] = true; - break; - } - case VK_VENDOR_ID_APPLE: - device->mul_mat_l[i] = false; - device->mul_mat_m[i] = true; - device->mul_mat_s[i] = false; - device->mul_mat_id_l[i] = false; - device->mul_mat_id_m[i] = true; - device->mul_mat_id_s[i] = false; - break; - case VK_VENDOR_ID_QUALCOMM: - device->mul_mat_l[i] = false; - device->mul_mat_m[i] = true; - device->mul_mat_s[i] = true; - device->mul_mat_id_l[i] = false; - device->mul_mat_id_m[i] = true; - device->mul_mat_id_s[i] = true; - break; -#endif - default: - device->mul_mat_l[i] = true; - device->mul_mat_m[i] = true; - device->mul_mat_s[i] = true; - device->mul_mat_id_l[i] = true; - device->mul_mat_id_m[i] = true; - device->mul_mat_id_s[i] = true; - break; - } + switch (a_type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_TQ1_0: + case GGML_TYPE_TQ2_0: + break; + default: + return nullptr; + } -#if VK_HEADER_VERSION >= 287 - // Honeykrisp driver for Asahi Linux doesn't report VK_VENDOR_ID_APPLE. - // Check for Honeykrisp driver and force same configuration as the VK_VENDOR_ID_APPLE case. - if (device->driver_id == vk::DriverId::eMesaHoneykrisp) { - device->mul_mat_l[i] = false; - device->mul_mat_m[i] = true; - device->mul_mat_s[i] = false; - device->mul_mat_id_l[i] = false; - device->mul_mat_id_m[i] = true; - device->mul_mat_id_s[i] = false; + // heuristic to choose workgroup size + uint32_t dmmv_wg = DMMV_WG_SIZE_SUBGROUP; + if ((ctx->device->vendor_id == VK_VENDOR_ID_NVIDIA && ctx->device->architecture != vk_device_architecture::NVIDIA_PRE_TURING) || ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { + // NOTE (2026-08-17): tried adding AMD_GCN here (DMMV_WG_SIZE_LARGE for the + // MoE expert mat-vec) -- MEASURED a REGRESSION on gfx803/RX480: 9.74 -> + // 13.63 ms/req. The larger workgroup's barrier/tree-reduction overhead + // outweighs the extra parallelism on Polaris. GCN stays on SUBGROUP. + // Prefer larger workgroups when M is small, to spread the work out more + // and keep more SMs busy. + // q6_k seems to prefer small workgroup size even for "medium" values of M. + if (a_type == GGML_TYPE_Q6_K) { + if (m < 4096 && k >= 1024) { + dmmv_wg = DMMV_WG_SIZE_LARGE; + } + } else { + if (m <= 8192 && k >= 1024) { + dmmv_wg = DMMV_WG_SIZE_LARGE; } -#endif - - device->mul_mat_l_int[i] = device->mul_mat_l[i]; - device->mul_mat_m_int[i] = device->mul_mat_m[i]; - device->mul_mat_s_int[i] = device->mul_mat_s[i]; - device->mul_mat_id_l_int[i] = device->mul_mat_id_l[i]; - device->mul_mat_id_m_int[i] = device->mul_mat_id_m[i]; - device->mul_mat_id_s_int[i] = device->mul_mat_id_s[i]; } + } - - std::vector<vk::DescriptorSetLayoutBinding> dsl_binding; - std::vector<vk::DescriptorBindingFlags> dsl_binding_flags; - for (uint32_t i = 0; i < MAX_PARAMETER_COUNT; i++) { - dsl_binding.push_back({i, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute}); - dsl_binding_flags.push_back({}); + if (b_type == GGML_TYPE_Q8_1) { + if (ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { + dmmv_wg = DMMV_WG_SIZE_SUBGROUP; } + return ctx->device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[dmmv_wg][a_type]; + } - vk::DescriptorSetLayoutBindingFlagsCreateInfo dslbfci = { dsl_binding_flags }; - - vk::DescriptorSetLayoutCreateInfo descriptor_set_layout_create_info( - {}, - dsl_binding); - descriptor_set_layout_create_info.setPNext(&dslbfci); - device->dsl = device->device.createDescriptorSetLayout(descriptor_set_layout_create_info); - - ggml_vk_load_shaders(device); - GGML_LOG_INFO("ggml_vulkan: %s multi_add=%d rte_fp16=%d\n", - device->name.c_str(), (int) device->multi_add, - (int) device->float_controls_rte_fp16); - - // Prefer a dedicated transfer queue on AMD dGPUs (non-GCN) when graphics queue use is disabled. - const bool prefers_transfer_queue = - device->vendor_id == VK_VENDOR_ID_AMD && - device->architecture != AMD_GCN && - !device->uma && - !allow_graphics_queue; - - if (!device->single_queue) { - const uint32_t transfer_queue_index = compute_queue_family_index == transfer_queue_family_index ? 1 : 0; - device->transfer_queue = ggml_vk_create_queue(device, transfer_queue_family_index, transfer_queue_index, { vk::PipelineStageFlagBits::eTransfer }, true); + return ctx->device->pipeline_dequant_mul_mat_vec_id_f32[dmmv_wg][a_type]; +} - device->async_use_transfer_queue = prefers_transfer_queue || (getenv("GGML_VK_ASYNC_USE_TRANSFER_QUEUE") != nullptr); - } else { - device->transfer_queue = ggml_vk_create_aliased_queue(device, device->compute_queue); +vk_subbuffer ggml_vk_tensor_subbuffer( + const ggml_backend_vk_context * ctx, const ggml_tensor * tensor, bool allow_misalign) { - device->async_use_transfer_queue = false; - } + vk_buffer buffer = nullptr; + size_t offset = 0; + if (ctx->device->uma) { + ggml_vk_host_get(ctx->device, tensor->data, buffer, offset); + } + if (!buffer) { + auto buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context; + buffer = buf_ctx->dev_buffer; + offset = vk_tensor_offset(tensor) + tensor->view_offs; + } + GGML_ASSERT(buffer != nullptr); - device->buffer_type = { - /* .iface = */ ggml_backend_vk_buffer_type_interface, - /* .device = */ ggml_backend_reg_dev_get(ggml_backend_vk_reg(), idx), - /* .context = */ new ggml_backend_vk_buffer_type_context{ device->name, device }, - }; + size_t size = ggml_nbytes(tensor); - device->fence = device->device.createFence({}); + const size_t descriptor_offset = ggml_vk_descriptor_offset( + offset, ctx->device->properties.limits.minStorageBufferOffsetAlignment, ggml_type_size(tensor->type)); + const size_t misalign_bytes = offset - descriptor_offset; + // The shader must support misaligned offsets when indexing into the buffer + GGML_ASSERT(allow_misalign || misalign_bytes == 0); + offset = descriptor_offset; + size += misalign_bytes; - device->idx = idx; + return vk_subbuffer{buffer, offset, size, tensor, allow_misalign}; +} - device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr; +static vk_subbuffer ggml_vk_mark_dynamic_subbuffer( + const ggml_backend_vk_context * ctx, vk_subbuffer buffer, const ggml_tensor * tensor) { + if (ctx->recording_plan != nullptr) { + buffer.tensor = tensor; + } + return buffer; +} - device->disable_fusion = getenv("GGML_VK_DISABLE_FUSION") != nullptr; +static void ggml_vk_graph_plan_update_descriptors( + ggml_backend_vk_context * ctx, ggml_backend_vk_graph_plan * plan) { + bool has_dynamic_binding = false; + for (const auto & update : plan->descriptor_updates) { + has_dynamic_binding = has_dynamic_binding || std::any_of( + update.sources.begin(), update.sources.end(), + [](const vk_subbuffer & source) { return source.tensor != nullptr; }); + } + if (!has_dynamic_binding) { + return; + } - device->add_rms_fusion = !device->disable_fusion && - device->subgroup_arithmetic && - device->vendor_id != VK_VENDOR_ID_INTEL; - device->partials_binding_alignment = - std::max(4u, (uint32_t)device->properties.limits.minStorageBufferOffsetAlignment); + std::vector<std::vector<vk::DescriptorBufferInfo>> infos; + std::vector<vk::WriteDescriptorSet> writes; + infos.reserve(plan->descriptor_updates.size()); + writes.reserve(plan->descriptor_updates.size()); + for (const auto & update : plan->descriptor_updates) { + infos.emplace_back(); + infos.back().reserve(update.sources.size()); + for (const vk_subbuffer & source : update.sources) { + const vk_subbuffer current = source.tensor != nullptr + ? ggml_vk_tensor_subbuffer(ctx, source.tensor, source.allow_misalign) + : source; + infos.back().push_back(current); + } + writes.push_back({ + update.descriptor_set, 0, 0, (uint32_t) infos.back().size(), + vk::DescriptorType::eStorageBuffer, nullptr, infos.back().data() + }); + } + ctx->device->device.updateDescriptorSets(writes, {}); +} - device->mmvq_mode = 0; - if (getenv("GGML_VK_DISABLE_MMVQ")) { - device->mmvq_mode = -1; - } else if (getenv("GGML_VK_FORCE_MMVQ")) { - device->mmvq_mode = 1; +static vk_command_buffer* ggml_vk_get_or_create_cmd_buffer(vk_device& device, vk_command_pool& pool) { + for (auto& cmd_buffer : pool.cmd_buffers) { + if (!cmd_buffer.in_use) { + cmd_buffer.use_counter++; + cmd_buffer.in_use = true; + return &cmd_buffer; } + } + return ggml_vk_create_cmd_buffer(device, pool); +} - return device; +static vk_submission ggml_vk_begin_submission(vk_device& device, vk_command_pool& p, bool one_time = true) { + vk_submission s; + s.buffer = ggml_vk_get_or_create_cmd_buffer(device, p); + if (one_time) { + s.buffer->buf.begin({ vk::CommandBufferUsageFlagBits::eOneTimeSubmit }); + } else { + s.buffer->buf.begin({ vk::CommandBufferUsageFlags{} }); } - return vk_instance.devices[idx]; + return s; } -static void ggml_vk_print_gpu_info(size_t idx) { - GGML_ASSERT(idx < vk_instance.device_indices.size()); - size_t dev_num = vk_instance.device_indices[idx]; - VK_LOG_DEBUG("ggml_vk_print_gpu_info(" << dev_num << ")"); - GGML_ASSERT(vk_instance_initialized); - - std::vector<vk::PhysicalDevice> devices = vk_instance.instance.enumeratePhysicalDevices(); +void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name) { + vk::DebugUtilsLabelEXT label = {}; + label.pLabelName = name; + label.color = std::array<float, 4>{1.0f, 1.0f, 1.0f, 1.0f}; + vk_instance.pfn_vkCmdBeginDebugUtilsLabelEXT(buf, reinterpret_cast<VkDebugUtilsLabelEXT *>(&label)); +} - if (dev_num >= devices.size()) { - std::cerr << "ggml_vulkan: Device with index " << dev_num << " does not exist." << std::endl; - throw std::runtime_error("Device not found"); +static void ggml_vk_flush_cpy_regions(vk_context& ctx) { + if (ctx == nullptr || ctx->cpy_regions.empty()) { + return; } + GGML_ASSERT(ctx->s != nullptr); + GGML_ASSERT(ctx->cpy_src != nullptr && ctx->cpy_dst != nullptr); + ctx->s->buffer->buf.copyBuffer(ctx->cpy_src->buffer, ctx->cpy_dst->buffer, ctx->cpy_regions); + ctx->cpy_regions.clear(); + ctx->cpy_src.reset(); + ctx->cpy_dst.reset(); +} - vk::PhysicalDevice physical_device = devices[dev_num]; - std::vector<vk::ExtensionProperties> ext_props = physical_device.enumerateDeviceExtensionProperties(); +void ggml_vk_ctx_end(vk_context& ctx) { + VK_LOG_DEBUG("ggml_vk_ctx_end(" << ctx << ", " << ctx->seqs.size() << ")"); + if (ctx->s == nullptr) { + return; + } - bool fp16_storage = false; - bool fp16_compute = false; - bool coopmat_support = false; - bool coopmat2_support = false; - bool coopmat2_decode_vector_support = false; - bool integer_dot_product = false; - bool bfloat16_support = false; - bool dot2_f16_support = false; - bool ocp_microscaling_extension = false; - bool shader_float8_extension = false; + ggml_vk_flush_cpy_regions(ctx); - for (auto properties : ext_props) { - if (strcmp("VK_KHR_16bit_storage", properties.extensionName) == 0) { - fp16_storage = true; - } else if (strcmp("VK_KHR_shader_float16_int8", properties.extensionName) == 0) { - fp16_compute = true; -#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - } else if (strcmp("VK_KHR_cooperative_matrix", properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_COOPMAT")) { - coopmat_support = true; -#endif -#if defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) - } else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_COOPMAT2")) { - coopmat2_support = true; -#endif - } else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) { - coopmat2_decode_vector_support = true; -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - } else if (strcmp("VK_KHR_shader_integer_dot_product", properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_INTEGER_DOT_PRODUCT")) { - integer_dot_product = true; -#endif -#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - } else if (strcmp("VK_KHR_shader_bfloat16", properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_BFLOAT16")) { - bfloat16_support = true; -#endif -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) - } else if (strcmp(VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME, properties.extensionName) == 0) { - ocp_microscaling_extension = true; -#endif -#if defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - } else if (strcmp(VK_EXT_SHADER_FLOAT8_EXTENSION_NAME, properties.extensionName) == 0) { - shader_float8_extension = true; -#endif - } else if (strcmp("VK_VALVE_shader_mixed_float_dot_product", properties.extensionName) == 0 && - !getenv("GGML_VK_DISABLE_DOT2")) { - dot2_f16_support = true; + // close open labels so this buffer is balanced; reopened in ggml_vk_ctx_begin + if (vk_instance.debug_utils_support) { + for (size_t i = 0; i < ctx->debug_labels.size(); i++) { + vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(ctx->s->buffer->buf); } + // the enclosing per-command-buffer region + vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(ctx->s->buffer->buf); } - const vk_device_architecture device_architecture = get_device_architecture(physical_device); - - const char* GGML_VK_DISABLE_F16 = getenv("GGML_VK_DISABLE_F16"); - bool force_disable_f16 = GGML_VK_DISABLE_F16 != nullptr; + ctx->s->buffer->buf.end(); + ctx->s = nullptr; +} - bool fp16 = !force_disable_f16 && fp16_storage && fp16_compute; +void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx, bool one_time) { + VK_LOG_DEBUG("ggml_vk_ctx_begin(" << device->name << ")"); + if (subctx->s != nullptr) { + ggml_vk_ctx_end(subctx); + } - vk::PhysicalDeviceProperties2 props2; - vk::PhysicalDeviceMaintenance3Properties props3; - vk::PhysicalDeviceSubgroupProperties subgroup_props; - vk::PhysicalDeviceDriverProperties driver_props; - vk::PhysicalDeviceShaderIntegerDotProductPropertiesKHR shader_integer_dot_product_props; - props2.pNext = &props3; - props3.pNext = &subgroup_props; - subgroup_props.pNext = &driver_props; + subctx->seqs.push_back({ ggml_vk_begin_submission(device, *subctx->p, one_time) }); + subctx->s = subctx->seqs[subctx->seqs.size() - 1].data(); - // Pointer to the last chain element - VkBaseOutStructure * last_struct = (VkBaseOutStructure *)&driver_props; + if (vk_instance.debug_utils_support) { + // outermost region, one per command buffer, so the gaps between submits stand out + const std::string name = "submit " + std::to_string(device->debug_cmdbuf_idx++); + ggml_vk_cmd_label_begin(subctx->s->buffer->buf, name.c_str()); - if (integer_dot_product) { - last_struct->pNext = (VkBaseOutStructure *)&shader_integer_dot_product_props; - last_struct = (VkBaseOutStructure *)&shader_integer_dot_product_props; + // reopen labels left open when the previous command buffer was submitted + for (const std::string & label : subctx->debug_labels) { + ggml_vk_cmd_label_begin(subctx->s->buffer->buf, label.c_str()); + } } +} - physical_device.getProperties2(&props2); - - VkPhysicalDeviceFeatures2 device_features2; - device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; - device_features2.pNext = nullptr; +vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) { + vk_context result; + if (!ctx->compute_ctx.expired()) { + result = ctx->compute_ctx.lock(); + } else { + if (ctx->recording_plan != nullptr) { + result = ggml_vk_create_temporary_context(ctx->recording_plan->command_pool); + ctx->gc.contexts.emplace_back(result); + } else { + result = ggml_vk_create_context(ctx, ctx->compute_cmd_pool); + } - VkPhysicalDeviceVulkan11Features vk11_features; - vk11_features.pNext = nullptr; - vk11_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES; - device_features2.pNext = &vk11_features; + ctx->compute_ctx = result; + ggml_vk_ctx_begin(ctx->device, result, ctx->recording_plan == nullptr); + } - VkPhysicalDeviceVulkan12Features vk12_features; - vk12_features.pNext = nullptr; - vk12_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; - vk11_features.pNext = &vk12_features; + if (ctx->recording_plan == nullptr && ctx->device->async_use_transfer_queue && + ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) { + result->s->wait_semaphores.push_back(ctx->transfer_semaphore); + ctx->transfer_semaphore_last_submitted = ctx->transfer_semaphore.value; + } - // Pointer to the last chain element - last_struct = (VkBaseOutStructure *)&vk12_features; + return result; +} -#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - VkPhysicalDeviceCooperativeMatrixFeaturesKHR coopmat_features; - coopmat_features.pNext = nullptr; - coopmat_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR; - coopmat_features.cooperativeMatrix = VK_FALSE; +vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx) { + vk_context result; + if (!ctx->transfer_ctx.expired()) { + result = ctx->transfer_ctx.lock(); + } else { + result = ggml_vk_create_context(ctx, ctx->transfer_cmd_pool); - if (coopmat_support) { - last_struct->pNext = (VkBaseOutStructure *)&coopmat_features; - last_struct = (VkBaseOutStructure *)&coopmat_features; + ctx->transfer_ctx = result; + ggml_vk_ctx_begin(ctx->device, result); } -#endif - VkPhysicalDeviceShaderIntegerDotProductFeaturesKHR shader_integer_dot_product_features {}; - shader_integer_dot_product_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_INTEGER_DOT_PRODUCT_FEATURES_KHR; - if (integer_dot_product) { - last_struct->pNext = (VkBaseOutStructure *)&shader_integer_dot_product_features; - last_struct = (VkBaseOutStructure *)&shader_integer_dot_product_features; - } + return result; +} -#if defined(VK_KHR_shader_bfloat16) - VkPhysicalDeviceShaderBfloat16FeaturesKHR bfloat16_features {}; - bfloat16_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR; - if (bfloat16_support) { - last_struct->pNext = (VkBaseOutStructure *)&bfloat16_features; - last_struct = (VkBaseOutStructure *)&bfloat16_features; +bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx) { + if (!ctx->device->async_use_transfer_queue || ctx->transfer_ctx.expired()) { + return false; } -#endif -#if defined(VK_NV_cooperative_matrix2) - VkPhysicalDeviceCooperativeMatrix2FeaturesNV coopmat2_features {}; - coopmat2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_2_FEATURES_NV; - if (coopmat2_support) { - last_struct->pNext = (VkBaseOutStructure *)&coopmat2_features; - last_struct = (VkBaseOutStructure *)&coopmat2_features; - } -#endif + vk_context cpy_ctx = ctx->transfer_ctx.lock(); + ggml_vk_ctx_end(cpy_ctx); - VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV coopmat2_decode_vector_features {}; - coopmat2_decode_vector_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV; - if (coopmat2_decode_vector_support) { - last_struct->pNext = (VkBaseOutStructure *)&coopmat2_decode_vector_features; - last_struct = (VkBaseOutStructure *)&coopmat2_decode_vector_features; + for (auto& cpy : cpy_ctx->in_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); } - VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE dot2_features {}; - dot2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE; - if (dot2_f16_support) { - last_struct->pNext = (VkBaseOutStructure *)&dot2_features; - last_struct = (VkBaseOutStructure *)&dot2_features; + ctx->transfer_semaphore.value++; + cpy_ctx->seqs.back().back().signal_semaphores.push_back(ctx->transfer_semaphore); + + ggml_vk_submit(cpy_ctx, {}); + ctx->transfer_ctx.reset(); + return true; +} + +static bool ggml_vk_submit_graph_plan( + ggml_backend_vk_context * ctx, ggml_backend_vk_graph_plan * plan) { + if (plan == nullptr || plan->owner != ctx || plan->unsupported || + plan->contexts.empty()) { + return false; } -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT ocp_microscaling_features {}; - ocp_microscaling_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OCP_MICROSCALING_TYPES_FEATURES_EXT; - VkPhysicalDeviceShaderFloat8FeaturesEXT shader_float8_features {}; - shader_float8_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT; - if (ocp_microscaling_extension) { - last_struct->pNext = (VkBaseOutStructure *)&ocp_microscaling_features; - last_struct = (VkBaseOutStructure *)&ocp_microscaling_features; + std::vector<vk::CommandBuffer> command_buffers; + for (const vk_context & subctx : plan->contexts) { + if (subctx == nullptr || subctx->seqs.size() != 1) { + return false; + } + for (const vk_submission & submission : subctx->seqs.front()) { + if (submission.buffer == nullptr || + !submission.wait_semaphores.empty() || + !submission.signal_semaphores.empty()) { + return false; + } + command_buffers.push_back(submission.buffer->buf); + } } - if (shader_float8_extension) { - last_struct->pNext = (VkBaseOutStructure *)&shader_float8_features; - last_struct = (VkBaseOutStructure *)&shader_float8_features; + if (command_buffers.empty()) { + return false; } -#endif - vkGetPhysicalDeviceFeatures2(physical_device, &device_features2); + vk::Semaphore wait_semaphore; + uint64_t wait_value = 0; + vk::PipelineStageFlags wait_stage; + if (ctx->device->async_use_transfer_queue && + ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) { + wait_semaphore = ctx->transfer_semaphore.s; + wait_value = ctx->transfer_semaphore.value; + wait_stage = ctx->device->compute_queue->stage_flags; + } - fp16 = fp16 && vk12_features.shaderFloat16; + std::vector<vk::SubmitInfo> submit_infos(command_buffers.size()); + for (size_t i = 0; i < command_buffers.size(); ++i) { + submit_infos[i] = { + 0, nullptr, nullptr, + 1, &command_buffers[i], + 0, nullptr, + }; + } -#if defined(VK_KHR_shader_bfloat16) - bool bf16 = bfloat16_support && bfloat16_features.shaderBFloat16Type; -#else - bool bf16 = false; -#endif - - uint32_t default_subgroup_size = get_subgroup_size("", device_architecture); - const size_t subgroup_size = (default_subgroup_size != 0) ? default_subgroup_size : subgroup_props.subgroupSize; - const bool uma = props2.properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; + vk::TimelineSemaphoreSubmitInfo timeline_info{}; + if (wait_semaphore) { + timeline_info = { 1, &wait_value, 0, nullptr }; + submit_infos.front() = { + 1, &wait_semaphore, &wait_stage, + 1, &command_buffers.front(), + 0, nullptr, + }; + submit_infos.front().setPNext(&timeline_info); + ctx->transfer_semaphore_last_submitted = wait_value; + } - integer_dot_product = integer_dot_product - && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated - && shader_integer_dot_product_features.shaderIntegerDotProduct; + ctx->device->compute_queue->handle->submit(submit_infos, {}); + ctx->submit_pending = true; + return true; +} - coopmat_support = coopmat_support -#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - && coopmat_features.cooperativeMatrix -#endif - && ggml_vk_khr_cooperative_matrix_support(props2.properties, driver_props, device_architecture); +size_t ggml_vk_align_size(size_t width, size_t align) { + VK_LOG_DEBUG("ggml_vk_align_size(" << width << ", " << align << ")"); + return CEIL_DIV(width, align) * align; +} -#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) - coopmat2_support = coopmat2_support && - coopmat2_features.cooperativeMatrixWorkgroupScope && - coopmat2_features.cooperativeMatrixFlexibleDimensions && - coopmat2_features.cooperativeMatrixReductions && - coopmat2_features.cooperativeMatrixConversions && - coopmat2_features.cooperativeMatrixPerElementOperations && - coopmat2_features.cooperativeMatrixTensorAddressing && - coopmat2_features.cooperativeMatrixBlockLoads; -#else - coopmat2_support = false; -#endif +void deferred_memcpy(void * dst, const void * src, size_t size, std::vector<vk_staging_memcpy>* memcpys) { + if (memcpys == nullptr) { + memcpy(dst, src, size); + } else { + memcpys->emplace_back(dst, src, size); + } +} - coopmat2_decode_vector_support = coopmat2_decode_vector_support && coopmat2_decode_vector_features.cooperativeMatrixDecodeVector; -#if !defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT) - coopmat2_decode_vector_support = false; -#endif +void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_staging_memset>* memsets) { + if (memsets == nullptr) { + memset(dst, val, size); + } else { + memsets->emplace_back(dst, val, size); + } +} - std::string matrix_cores = coopmat2_support ? (coopmat2_decode_vector_support ? "NV_coopmat2v" : "NV_coopmat2") - : coopmat_support ? "KHR_coopmat" - : "none"; +struct ggml_vk_wp_event { + vk_device device; + vk_context context; + vk::Fence fence; +}; - bool dot2_f16 = dot2_f16_support && dot2_features.shaderMixedFloatDotProductFloat16AccFloat32; - const char *fp16_str = fp16 ? (dot2_f16 ? "dot2" : "1") : "0"; -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - const bool fp4 = ocp_microscaling_extension && ocp_microscaling_features.shaderFloat4 && - shader_float8_extension && shader_float8_features.shaderFloat8 && - !getenv("GGML_VK_DISABLE_OCP_FP4"); -#else - GGML_UNUSED(ocp_microscaling_extension); - GGML_UNUSED(shader_float8_extension); - const bool fp4 = false; -#endif +// GGML_VK_WP_HASHCHK=1 debug scaffold. GGML_VK_WP_VERIFY proves the bytes are +// right when they are STAGED; it says nothing about whether they are still right +// when the dispatch that reads them is recorded. This registry closes that gap: +// stage_in records a hash per pool offset, and the paged mul_mat_id dispatch +// re-reads the bound pool at the offset it is about to hand the shader and +// compares. A mismatch means the slot was overwritten between stage-in and +// dispatch, i.e. a slot-lifetime bug. All matching means the data the shader +// reads is correct and the fault is elsewhere. +static std::mutex g_wp_hash_mutex; +static std::unordered_map<size_t, uint64_t> g_wp_slot_hash; - std::string device_name = props2.properties.deviceName.data(); - GGML_LOG_DEBUG("ggml_vulkan: %zu = %s (%s) | uma: %d | fp16: %s | bf16: %d | fp4: %d | warp size: %zu | shared memory: %d | int dot: %d | matrix cores: %s\n", - idx, device_name.c_str(), driver_props.driverName.data(), uma, fp16_str, bf16, fp4, subgroup_size, - props2.properties.limits.maxComputeSharedMemorySize, integer_dot_product, matrix_cores.c_str()); +static bool wp_hashchk_enabled() { + static const bool v = getenv("GGML_VK_WP_HASHCHK") != nullptr; + return v; +} - if (props2.properties.deviceType == vk::PhysicalDeviceType::eCpu) { - GGML_LOG_DEBUG("ggml_vulkan: Warning: Device type is CPU. This is probably not the device you want.\n"); +static uint64_t wp_hash_bytes(const void * p, size_t n) { + // FNV-1a; only needs to detect change, not resist anything. + uint64_t h = 0xcbf29ce484222325ull; + const uint8_t * b = (const uint8_t *) p; + for (size_t i = 0; i < n; ++i) { + h ^= b[i]; + h *= 0x100000001b3ull; } + return h; } -static bool ggml_vk_instance_layer_settings_available(); -static bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions); -static bool ggml_vk_instance_debug_utils_ext_available(const std::vector<vk::ExtensionProperties> & instance_extensions); -static bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev); - -static DispatchLoaderDynamic ggml_vk_default_dispatcher_instance; -DispatchLoaderDynamic & ggml_vk_default_dispatcher() { - return ggml_vk_default_dispatcher_instance; -} +static constexpr size_t WP_HASH_SPAN = 4096; -static void ggml_vk_instance_init() { - if (vk_instance_initialized) { - return; +bool ggml_backend_vk_wp_stage_in(ggml_backend_buffer_t buffer, + void * dst, const void * src, + size_t payload_size, size_t slot_size, + void ** event) { + if (buffer == nullptr || dst == nullptr || src == nullptr || event == nullptr || payload_size > slot_size) { + static std::atomic<bool> warned{false}; + if (!warned.exchange(true)) { + GGML_LOG_WARN("ggml_backend_vk_wp_stage_in: invalid arguments " + "(buffer=%p dst=%p src=%p event=%p payload=%zu slot=%zu)\n", + (void *) buffer, dst, src, (void *) event, payload_size, slot_size); + } + return false; } - VK_LOG_DEBUG("ggml_vk_instance_init()"); - // See https://github.com/KhronosGroup/Vulkan-Hpp?tab=readme-ov-file#extensions--per-device-function-pointers- - ggml_vk_default_dispatcher_instance.init(vkGetInstanceProcAddr); - - uint32_t api_version = vk::enumerateInstanceVersion(); + auto * buffer_ctx = (ggml_backend_vk_buffer_context *) buffer->context; + if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { + static std::atomic<bool> warned{false}; + if (!warned.exchange(true)) { + GGML_LOG_WARN("ggml_backend_vk_wp_stage_in: missing Vulkan buffer context " + "(context=%p dev_buffer=%p)\n", + (void *) buffer_ctx, + buffer_ctx == nullptr ? nullptr : (void *) buffer_ctx->dev_buffer.get()); + } + return false; + } - if (api_version < VK_API_VERSION_1_2) { - std::cerr << "ggml_vulkan: Error: Vulkan 1.2 required." << std::endl; - throw vk::SystemError(vk::Result::eErrorFeatureNotPresent, "Vulkan 1.2 required"); + vk_buffer dst_buffer = buffer_ctx->dev_buffer; + const uintptr_t base_addr = (uintptr_t) ggml_backend_buffer_get_base(buffer); + const uintptr_t dst_addr = (uintptr_t) dst; + const bool below_base = dst_addr < base_addr; + const size_t offset = below_base ? 0 : dst_addr - base_addr; + if (below_base || offset > dst_buffer->size || slot_size > dst_buffer->size - offset) { + static std::atomic<bool> warned{false}; + if (!warned.exchange(true)) { + GGML_LOG_WARN("ggml_backend_vk_wp_stage_in: destination outside pool buffer " + "(base=%p dst=%p offset=%zu slot=%zu buffer_size=%zu)\n", + (void *) base_addr, dst, offset, slot_size, dst_buffer->size); + } + return false; } - vk::ApplicationInfo app_info{ "ggml-vulkan", 1, nullptr, 0, api_version }; + std::lock_guard<std::recursive_mutex> guard(dst_buffer->device->mutex); - const std::vector<vk::ExtensionProperties> instance_extensions = vk::enumerateInstanceExtensionProperties(); - const bool layer_settings = ggml_vk_instance_layer_settings_available(); -#ifdef __APPLE__ - const bool portability_enumeration_ext = ggml_vk_instance_portability_enumeration_ext_available(instance_extensions); -#endif - const bool debug_utils_ext = ggml_vk_instance_debug_utils_ext_available(instance_extensions) && getenv("GGML_VK_DEBUG_MARKERS") != nullptr; - std::vector<const char*> layers; + // Can this transfer safely be left in flight? Only if `src` is host memory + // registered with THIS Vulkan device, in which case the copy reads straight + // from it. Otherwise ggml_vk_buffer_write_async routes through the single + // device-wide `device->sync_staging` buffer, and two overlapping transfers + // would each memcpy into that same region -- the second clobbering data the + // first's queued copy has not executed yet. So an unpinned source MUST be + // fenced before this function returns; the caller cannot know this, because + // the staging decision is made in here. + // + // The pager registers its O_DIRECT arena through + // ggml_backend_vk_wp_host_register. If import is unavailable, this lookup + // stays empty and the correctness-preserving fenced path remains active. + vk_buffer pinned_buf = nullptr; + size_t pinned_offset = 0; + ggml_vk_host_get(dst_buffer->device, src, pinned_buf, pinned_offset); + const bool src_is_pinned = pinned_buf != nullptr; - if (layer_settings) { - layers.push_back("VK_LAYER_KHRONOS_validation"); - } - std::vector<const char*> extensions; - if (layer_settings) { - extensions.push_back("VK_EXT_layer_settings"); + // GGML_VK_WP_CQUEUE=1: stage on the COMPUTE queue instead of the transfer + // queue. On this device the two are separate queues (same family, 4 queues on + // RADV Polaris), and nothing orders a transfer-queue submission against the + // compute-queue dispatches that read the slot. Same queue == guaranteed order. + static const bool wp_cqueue = getenv("GGML_VK_WP_CQUEUE") != nullptr; + vk_context context = ggml_vk_create_temporary_context( + wp_cqueue ? dst_buffer->device->compute_queue->cmd_pool + : dst_buffer->device->transfer_queue->cmd_pool); + ggml_vk_ctx_begin(dst_buffer->device, context); + if (!ggml_vk_buffer_write_async(context, dst_buffer, offset, src, payload_size, true)) { + static std::atomic<bool> warned{false}; + if (!warned.exchange(true)) { + GGML_LOG_WARN("ggml_backend_vk_wp_stage_in: ggml_vk_buffer_write_async returned false " + "(offset=%zu payload=%zu source_registered=%d)\n", + offset, payload_size, src_is_pinned ? 1 : 0); + } + return false; } -#ifdef __APPLE__ - if (portability_enumeration_ext) { - extensions.push_back("VK_KHR_portability_enumeration"); + if (slot_size > payload_size) { + ggml_vk_buffer_memset_async(context, dst_buffer, offset + payload_size, 0, slot_size - payload_size); } -#endif - if (debug_utils_ext) { - extensions.push_back("VK_EXT_debug_utils"); + ggml_vk_ctx_end(context); + + for (auto & copy : context->in_memcpys) { + memcpy(copy.dst, copy.src, copy.n); } - VkBool32 enable_best_practice = layer_settings; - std::vector<vk::LayerSettingEXT> settings = { - { - "VK_LAYER_KHRONOS_validation", - "validate_best_practices", - vk::LayerSettingTypeEXT::eBool32, - 1, - &enable_best_practice - }, - }; - vk::LayerSettingsCreateInfoEXT layer_setting_info(settings); - vk::InstanceCreateInfo instance_create_info(vk::InstanceCreateFlags{}, &app_info, layers, extensions, &layer_setting_info); -#ifdef __APPLE__ - if (portability_enumeration_ext) { - instance_create_info.flags |= vk::InstanceCreateFlagBits::eEnumeratePortabilityKHR; + for (auto & memset : context->memsets) { + ::memset(memset.dst, memset.val, memset.n); } -#endif - vk_instance.instance = vk::createInstance(instance_create_info); - vk_instance_initialized = true; + auto * wp_event = new ggml_vk_wp_event{ + dst_buffer->device, + context, + dst_buffer->device->device.createFence({}), + }; + ggml_vk_submit(context, wp_event->fence); - if (debug_utils_ext) { - vk_instance.debug_utils_support = true; - vk_instance.pfn_vkSetDebugUtilsObjectNameEXT = (PFN_vkSetDebugUtilsObjectNameEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkSetDebugUtilsObjectNameEXT"); - vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT = (PFN_vkQueueBeginDebugUtilsLabelEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkQueueBeginDebugUtilsLabelEXT"); - vk_instance.pfn_vkQueueEndDebugUtilsLabelEXT = (PFN_vkQueueEndDebugUtilsLabelEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkQueueEndDebugUtilsLabelEXT"); - vk_instance.pfn_vkCmdBeginDebugUtilsLabelEXT = (PFN_vkCmdBeginDebugUtilsLabelEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkCmdBeginDebugUtilsLabelEXT"); - vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT = (PFN_vkCmdEndDebugUtilsLabelEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkCmdEndDebugUtilsLabelEXT"); - vk_instance.pfn_vkCmdInsertDebugUtilsLabelEXT = (PFN_vkCmdInsertDebugUtilsLabelEXT) vkGetInstanceProcAddr(vk_instance.instance, "vkCmdInsertDebugUtilsLabelEXT"); + // Shared-staging transfers cannot be left in flight -- see the note above. + if (!src_is_pinned) { + (void) ggml_backend_vk_wp_event_wait(wp_event); } - vk_perf_logger_enabled = getenv("GGML_VK_PERF_LOGGER") != nullptr; - vk_perf_logger_concurrent = getenv("GGML_VK_PERF_LOGGER_CONCURRENT") != nullptr; - { - const char * e = getenv("WP_VK_NODE_TS"); - vk_node_ts_enabled = e != nullptr && e[0] == '1'; - if (vk_node_ts_enabled) { - GGML_LOG_INFO("ggml_vulkan: WP_VK_NODE_TS=1 timestamps harvested at synchronize\n"); - } - } - vk_enable_sync_logger = getenv("GGML_VK_SYNC_LOGGER") != nullptr; - vk_memory_logger_enabled = getenv("GGML_VK_MEMORY_LOGGER") != nullptr; - const char* GGML_VK_PIPELINE_STATS = getenv("GGML_VK_PIPELINE_STATS"); - if (GGML_VK_PIPELINE_STATS != nullptr) { - vk_pipeline_stats_filter = GGML_VK_PIPELINE_STATS; - } - const char* GGML_VK_PERF_LOGGER_FREQUENCY = getenv("GGML_VK_PERF_LOGGER_FREQUENCY"); - - if (GGML_VK_PERF_LOGGER_FREQUENCY != nullptr) { - vk_perf_logger_frequency = std::stoul(GGML_VK_PERF_LOGGER_FREQUENCY); + if (wp_hashchk_enabled()) { + const size_t n = std::min<size_t>(payload_size, WP_HASH_SPAN); + std::lock_guard<std::mutex> hg(g_wp_hash_mutex); + g_wp_slot_hash[offset] = wp_hash_bytes(src, n); } - // See https://github.com/KhronosGroup/Vulkan-Hpp?tab=readme-ov-file#extensions--per-device-function-pointers- - VULKAN_HPP_DEFAULT_DISPATCHER.init(vk_instance.instance); - - std::vector<vk::PhysicalDevice> devices = vk_instance.instance.enumeratePhysicalDevices(); - - // Emulate behavior of CUDA_VISIBLE_DEVICES for Vulkan - char * devices_env = getenv("GGML_VK_VISIBLE_DEVICES"); - if (devices_env != nullptr) { - size_t num_available_devices = devices.size(); - - std::string devices(devices_env); - std::replace(devices.begin(), devices.end(), ',', ' '); - - std::stringstream ss(devices); - size_t tmp; - while (ss >> tmp) { - if(tmp >= num_available_devices) { - std::cerr << "ggml_vulkan: Invalid device index " << tmp << " in GGML_VK_VISIBLE_DEVICES." << std::endl; - throw std::runtime_error("Invalid Vulkan device index"); + // GGML_VK_WP_VERIFY=1: read the slot straight back and compare with the + // source. Answers "did the bytes actually land" without inference in the + // way. Debug only -- it stalls the queue on every page. + static const bool wp_verify = getenv("GGML_VK_WP_VERIFY") != nullptr; + if (wp_verify) { + (void) ggml_backend_vk_wp_event_wait(wp_event); + static int checked = 0, bad = 0; + const size_t n = std::min<size_t>(payload_size, 4096); + std::vector<uint8_t> back(n); + ggml_vk_buffer_read(dst_buffer, offset, back.data(), n); + if (memcmp(back.data(), src, n) != 0) { + if (bad < 4) { + size_t i = 0; + while (i < n && back[i] == ((const uint8_t *) src)[i]) { ++i; } + fprintf(stderr, "[vk-wp-verify] MISMATCH at slot offset %zu, first bad byte %zu: " + "got 0x%02x want 0x%02x (payload %zu)\n", + offset, i, back[i], ((const uint8_t *) src)[i], payload_size); } - vk_instance.device_indices.push_back(tmp); + ++bad; } - } else { - // If no vulkan devices are found, return early - if (devices.empty()) { - GGML_LOG_INFO("ggml_vulkan: No devices found.\n"); - return; + if (++checked % 500 == 0) { + fprintf(stderr, "[vk-wp-verify] %d pages checked, %d mismatched\n", checked, bad); } + } - // Default to using all dedicated GPUs - for (size_t i = 0; i < devices.size(); i++) { - vk::PhysicalDeviceProperties2 new_props; - vk::PhysicalDeviceDriverProperties new_driver; - vk::PhysicalDeviceIDProperties new_id; - new_props.pNext = &new_driver; - new_driver.pNext = &new_id; - devices[i].getProperties2(&new_props); - - if ((new_props.properties.deviceType == vk::PhysicalDeviceType::eDiscreteGpu || new_props.properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu) && ggml_vk_device_is_supported(devices[i])) { - // Check if there are two physical devices corresponding to the same GPU - // This handles the case where the same GPU appears with different drivers (e.g., RADV + AMDVLK on Linux), - // see https://github.com/ggml-org/llama.cpp/pull/7582 for original deduplication. - // MoltenVK on macOS may report the same UUID for distinct GPUs on multi-GPU cards, - // see https://github.com/KhronosGroup/MoltenVK/issues/2683. Skip when both old/new - // driver is MoltenVK - auto old_device = std::find_if( - vk_instance.device_indices.begin(), - vk_instance.device_indices.end(), - [&devices, &new_id, &new_driver](const size_t k){ - vk::PhysicalDeviceProperties2 old_props; - vk::PhysicalDeviceDriverProperties old_driver; - vk::PhysicalDeviceIDProperties old_id; - old_props.pNext = &old_driver; - old_driver.pNext = &old_id; - devices[k].getProperties2(&old_props); - - bool same_uuid = std::equal(std::begin(old_id.deviceUUID), std::end(old_id.deviceUUID), std::begin(new_id.deviceUUID)); - same_uuid = same_uuid || ( - old_id.deviceLUIDValid && new_id.deviceLUIDValid && - std::equal(std::begin(old_id.deviceLUID), std::end(old_id.deviceLUID), std::begin(new_id.deviceLUID)) - ); - bool both_molten_vk = (new_driver.driverID == vk::DriverId::eMoltenvk && old_driver.driverID == vk::DriverId::eMoltenvk); - - return same_uuid && !both_molten_vk; - } - ); - if (old_device == vk_instance.device_indices.end()) { - vk_instance.device_indices.push_back(i); - } else { - // There can be two physical devices corresponding to the same GPU if there are 2 different drivers - // This can cause error when splitting layers aross the devices, need to keep only 1 - VK_LOG_DEBUG("Device " << i << " and device " << *old_device << " have the same deviceUUID"); - - vk::PhysicalDeviceProperties2 old_props; - vk::PhysicalDeviceDriverProperties old_driver; - old_props.pNext = &old_driver; - devices[*old_device].getProperties2(&old_props); - - std::map<vk::DriverId, int> driver_priorities {}; - int old_priority = std::numeric_limits<int>::max(); - int new_priority = std::numeric_limits<int>::max(); - - // Check https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkDriverId.html for the list of driver id - // Smaller number -> higher priority - switch (old_props.properties.vendorID) { - case VK_VENDOR_ID_AMD: - driver_priorities[vk::DriverId::eMesaRadv] = 1; - driver_priorities[vk::DriverId::eAmdOpenSource] = 2; - driver_priorities[vk::DriverId::eAmdProprietary] = 3; - break; - case VK_VENDOR_ID_INTEL: - driver_priorities[vk::DriverId::eIntelOpenSourceMESA] = 1; - driver_priorities[vk::DriverId::eIntelProprietaryWindows] = 2; - break; - case VK_VENDOR_ID_NVIDIA: - driver_priorities[vk::DriverId::eNvidiaProprietary] = 1; -#if defined(VK_API_VERSION_1_3) && VK_HEADER_VERSION >= 235 - driver_priorities[vk::DriverId::eMesaNvk] = 2; -#endif - break; - case VK_VENDOR_ID_QUALCOMM: - driver_priorities[vk::DriverId::eQualcommProprietary] = 1; - driver_priorities[vk::DriverId::eMesaTurnip] = 2; - break; - } - driver_priorities[vk::DriverId::eMesaDozen] = 100; + *event = wp_event; + return true; +} - if (driver_priorities.count(old_driver.driverID)) { - old_priority = driver_priorities[old_driver.driverID]; - } - if (driver_priorities.count(new_driver.driverID)) { - new_priority = driver_priorities[new_driver.driverID]; - } +void * ggml_backend_vk_wp_host_alloc(ggml_backend_buffer_t pool_buffer, size_t size) { + if (pool_buffer == nullptr || size == 0) { + return nullptr; + } + auto * buffer_ctx = (ggml_backend_vk_buffer_context *) pool_buffer->context; + if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { + return nullptr; + } + // Registered against the POOL's device, not vk_instance.devices[0], so + // ggml_vk_host_get finds it when staging into this pool. + return ggml_vk_host_malloc(buffer_ctx->dev_buffer->device, size); +} - if (new_priority < old_priority) { - auto r = std::remove(vk_instance.device_indices.begin(), vk_instance.device_indices.end(), *old_device); - vk_instance.device_indices.erase(r, vk_instance.device_indices.end()); - vk_instance.device_indices.push_back(i); +bool ggml_backend_vk_wp_host_register( + ggml_backend_buffer_t pool_buffer, void * ptr, size_t size) { + if (pool_buffer == nullptr || ptr == nullptr || size == 0) { + return false; + } + auto * buffer_ctx = (ggml_backend_vk_buffer_context *) pool_buffer->context; + if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { + return false; + } - VK_LOG_DEBUG("Prioritize device " << i << " driver " << new_driver.driverName << " over device " << *old_device << " driver " << old_driver.driverName); - } - else { - VK_LOG_DEBUG("Prioritize device " << *old_device << " driver " << old_driver.driverName << " over device " << i << " driver " << new_driver.driverName << std::endl); - } - } - } + vk_device device = buffer_ctx->dev_buffer->device; + if (!device->external_memory_host) { + static std::atomic<bool> warned{false}; + if (!warned.exchange(true)) { + GGML_LOG_WARN("ggml_backend_vk_wp_host_register: device does not support " + "VK_EXT_external_memory_host\n"); } - - // If no GPUs found, fall back to the first non-CPU device. - // If only CPU devices are available, return without devices. - if (vk_instance.device_indices.empty()) { - for (size_t i = 0; i < devices.size(); i++) { - if (devices[i].getProperties().deviceType != vk::PhysicalDeviceType::eCpu) { - vk_instance.device_indices.push_back(i); - break; - } - } + return false; + } + const size_t import_alignment = device->min_imported_host_pointer_alignment; + if (import_alignment == 0 || + ((uintptr_t) ptr & (import_alignment - 1)) != 0 || + (size & (import_alignment - 1)) != 0) { + static std::atomic<bool> warned{false}; + if (!warned.exchange(true)) { + GGML_LOG_WARN("ggml_backend_vk_wp_host_register: host allocation does not meet " + "Vulkan import alignment (ptr=%p size=%zu alignment=%zu)\n", + ptr, size, import_alignment); } + return false; + } - if (vk_instance.device_indices.empty()) { - GGML_LOG_INFO("ggml_vulkan: No devices found.\n"); - return; + vk_buffer imported = ggml_vk_buffer_from_host_ptr(device, ptr, size); + if (imported == nullptr) { + static std::atomic<bool> warned{false}; + if (!warned.exchange(true)) { + GGML_LOG_WARN("ggml_backend_vk_wp_host_register: Vulkan host pointer import failed " + "(ptr=%p size=%zu)\n", ptr, size); } + return false; } - GGML_LOG_DEBUG("ggml_vulkan: Found %zu Vulkan devices:\n", vk_instance.device_indices.size()); - for (size_t i = 0; i < vk_instance.device_indices.size(); i++) { - vk::PhysicalDevice vkdev = devices[vk_instance.device_indices[i]]; - std::vector<vk::ExtensionProperties> extensionprops = vkdev.enumerateDeviceExtensionProperties(); + const uintptr_t begin = (uintptr_t) ptr; + const uintptr_t end = begin + size; + if (end < begin) { + return false; + } - bool membudget_supported = false; - for (const auto & ext : extensionprops) { - if (strcmp(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME, ext.extensionName) == 0) { - membudget_supported = true; - break; - } + std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex); + for (const auto & entry : device->pinned_memory) { + const uintptr_t registered_begin = (uintptr_t) std::get<0>(entry); + const uintptr_t registered_end = registered_begin + std::get<1>(entry); + if (begin < registered_end && registered_begin < end) { + return false; } - - vk_instance.device_supports_membudget.push_back(membudget_supported); - - ggml_vk_print_gpu_info(i); } + device->pinned_memory.push_back(std::make_tuple(ptr, size, std::move(imported))); + return true; } -static void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { - VK_LOG_DEBUG("ggml_vk_init(" << ctx->name << ", " << idx << ")"); - ggml_vk_instance_init(); - GGML_ASSERT(idx < vk_instance.device_indices.size()); - - ctx->name = GGML_VK_NAME + std::to_string(idx); - - ctx->device = ggml_vk_get_device(idx); - - ctx->semaphore_idx = 0; - ctx->event_idx = 0; +void ggml_backend_vk_wp_host_unregister( + ggml_backend_buffer_t pool_buffer, void * ptr) { + if (pool_buffer == nullptr || ptr == nullptr) { + return; + } + auto * buffer_ctx = (ggml_backend_vk_buffer_context *) pool_buffer->context; + if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { + return; + } + ggml_vk_host_free(buffer_ctx->dev_buffer->device, ptr); +} - ctx->prealloc_size_x = 0; - ctx->prealloc_size_y = 0; - ctx->prealloc_size_split_k = 0; - // Fixed size of 1KB, for deterministic behavior - ctx->prealloc_size_add_rms_partials = 1024; +bool ggml_backend_vk_wp_read(ggml_backend_buffer_t pool_buffer, + const void * src, void * dst, size_t n) { + if (pool_buffer == nullptr || src == nullptr || dst == nullptr || n == 0) { + return false; + } + auto * buffer_ctx = (ggml_backend_vk_buffer_context *) pool_buffer->context; + if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { + return false; + } + vk_buffer src_buffer = buffer_ctx->dev_buffer; + const size_t offset = (uintptr_t) src - (uintptr_t) vk_ptr_base; + if (offset > src_buffer->size || n > src_buffer->size - offset) { + return false; + } + ggml_vk_buffer_read(src_buffer, offset, dst, n); + return true; +} - ctx->fence = ctx->device->device.createFence({}); - ctx->almost_ready_fence = ctx->device->device.createFence({}); +void ggml_backend_vk_wp_host_free(ggml_backend_buffer_t pool_buffer, void * ptr) { + if (pool_buffer == nullptr || ptr == nullptr) { + return; + } + auto * buffer_ctx = (ggml_backend_vk_buffer_context *) pool_buffer->context; + if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { + return; + } + ggml_vk_host_free(buffer_ctx->dev_buffer->device, ptr); +} - ctx->compute_cmd_pool.init(ctx->device, ctx->device->compute_queue.get()); - if (ctx->device->async_use_transfer_queue) { - vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eTimeline, 0 }; - vk::SemaphoreCreateInfo ci{}; - ci.setPNext(&tci); - ctx->transfer_semaphore.s = ctx->device->device.createSemaphore(ci); - ctx->transfer_semaphore.value = 0; +// Weight-paging consumption bridge. Mirrors the CUDA backend's +// ggml_cuda_set_routed_expert_ptrs: the pager publishes where the active +// experts actually live, and the very next mul_mat_id dispatch consumes it. +// Thread-local for the same reason CUDA's is — eval runs on the compute thread +// and a publication must not leak across threads. +struct vk_wp_expert_offsets { + ggml_backend_buffer_t pool_buffer = nullptr; + std::vector<uint32_t> offsets; + bool valid = false; +}; +static thread_local vk_wp_expert_offsets tls_wp_expert_offsets; - ctx->transfer_cmd_pool.init(ctx->device, ctx->device->transfer_queue.get()); +void ggml_backend_vk_wp_set_expert_offsets(ggml_backend_buffer_t pool_buffer, + const uint32_t * block_offsets, + int n_experts) { + if (pool_buffer == nullptr || block_offsets == nullptr || n_experts <= 0) { + tls_wp_expert_offsets.valid = false; + return; } + tls_wp_expert_offsets.pool_buffer = pool_buffer; + tls_wp_expert_offsets.offsets.assign(block_offsets, block_offsets + n_experts); + tls_wp_expert_offsets.valid = true; +} - if (vk_perf_logger_enabled) { - ctx->perf_logger = std::unique_ptr<vk_perf_logger>(new vk_perf_logger()); +bool ggml_backend_vk_wp_fused_expert( + ggml_backend_t backend, + const struct ggml_backend_vk_wp_fused_expert_params * params, + struct ggml_backend_vk_wp_fused_expert_stats * stats) { + if (stats != nullptr) { + *stats = {}; + } + if (backend == nullptr || !ggml_backend_is_vk(backend) || + params == nullptr || stats == nullptr || + params->io_buffer == nullptr || params->weights_buffers == nullptr || + params->params_buffer == nullptr || params->gate_byte_offsets == nullptr || + params->up_byte_offsets == nullptr || params->down_byte_offsets == nullptr || + params->route_weights == nullptr || params->n_tokens == 0 || + params->n_tokens > 8 || params->n_experts == 0 || params->n_ff == 0 || + params->n_ff > 256 || params->n_embd == 0 || + (params->down_type != GGML_TYPE_Q5_1 && params->down_type != GGML_TYPE_Q8_0)) { + return false; } -#ifdef GGML_VULKAN_CHECK_RESULTS - const char* skip_checks = getenv("GGML_VULKAN_SKIP_CHECKS"); - vk_skip_checks = (skip_checks == NULL ? 0 : atoi(skip_checks)); - const char* output_tensor = getenv("GGML_VULKAN_OUTPUT_TENSOR"); - vk_output_tensor = (output_tensor == NULL ? 0 : atoi(output_tensor)); -#endif -} - -static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type) { - VK_LOG_DEBUG("ggml_vk_get_to_fp16()"); - switch (type) { - case GGML_TYPE_F32: - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: - break; - default: - return nullptr; + auto * ctx = (ggml_backend_vk_context *) backend->context; + auto * io_ctx = (ggml_backend_vk_buffer_context *) params->io_buffer->context; + auto * params_ctx = (ggml_backend_vk_buffer_context *) params->params_buffer->context; + if (ctx == nullptr || io_ctx == nullptr || + params_ctx == nullptr || io_ctx->dev_buffer == nullptr || + params_ctx->dev_buffer == nullptr || + io_ctx->dev_buffer->device != ctx->device || + params_ctx->dev_buffer->device != ctx->device || + params->n_experts > 4096) { + return false; } - return ctx->device->pipeline_dequant[type]; -} + const size_t n_values = (size_t) params->n_tokens * params->n_embd; + if (n_values > std::numeric_limits<size_t>::max() / sizeof(float)) { + return false; + } + const size_t io_bytes = n_values * sizeof(float); + const size_t io_size = io_ctx->dev_buffer->size; + if (params->input_offset > io_size || io_bytes > io_size - params->input_offset || + params->output_offset > io_size || io_bytes > io_size - params->output_offset) { + return false; + } -static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_pipeline(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { - VK_LOG_DEBUG("ggml_vk_get_mul_mat_mat_pipeline(" << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ", " << prec << ")"); - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_f32; + const size_t gate_blocks = ((size_t) params->n_embd + 255) / 256; + const size_t down_blocks = ((size_t) params->n_ff + 31) / 32; + if (gate_blocks == 0 || down_blocks == 0 || + (size_t) params->n_ff > std::numeric_limits<size_t>::max() / gate_blocks || + (size_t) params->n_embd > std::numeric_limits<size_t>::max() / down_blocks) { + return false; } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_f32_f16; + const size_t gate_block_bytes = 144; + const size_t down_block_bytes = params->down_type == GGML_TYPE_Q8_0 ? 34 : 24; + // Bytes spanned by one expert's gate/up role (they share a shape) and by + // one expert's down role. Each role is bound as its OWN sub-buffer at + // its own byte offset in the dispatch loop below, so these are also the + // descriptor ranges -- NOT the whole weights buffer. + if ((uint64_t) params->n_ff > std::numeric_limits<uint64_t>::max() / gate_blocks || + (uint64_t) params->n_ff * gate_blocks > + std::numeric_limits<uint64_t>::max() / gate_block_bytes) { + return false; } - if (src0_type == GGML_TYPE_BF16 && src1_type == GGML_TYPE_BF16) { - return ctx->device->pipeline_matmul_bf16; + const uint64_t gate_role_bytes = (uint64_t) params->n_ff * gate_blocks * gate_block_bytes; + if ((uint64_t) params->n_embd > std::numeric_limits<uint64_t>::max() / down_blocks || + (uint64_t) params->n_embd * down_blocks > + std::numeric_limits<uint64_t>::max() / down_block_bytes) { + return false; } - if (prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support)) { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_f16_f32.f16acc; + const uint64_t down_role_bytes = (uint64_t) params->n_embd * down_blocks * down_block_bytes; + + // An expert slot's position inside the pool buffer is only guaranteed + // aligned to this value (see layout_sliced_pages()/arena_slot_stride() + // in wp-expert-worker.cpp, both of which key off + // ggml_backend_buft_get_alignment() == minStorageBufferOffsetAlignment + // for the Vulkan buffer type), not to the coarser quant block size. + const VkDeviceSize align = ctx->device->properties.limits.minStorageBufferOffsetAlignment; + + std::vector<vk_buffer> weight_buffers(params->n_experts); + std::vector<size_t> weight_sizes(params->n_experts); + for (uint32_t e = 0; e < params->n_experts; ++e) { + if (params->weights_buffers[e] == nullptr) { + return false; } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_f16.f16acc; + auto * weights_ctx = (ggml_backend_vk_buffer_context *) + params->weights_buffers[e]->context; + if (weights_ctx == nullptr || weights_ctx->dev_buffer == nullptr || + weights_ctx->dev_buffer->device != ctx->device) { + return false; } - } else { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_f16_f32.f32acc; + weight_buffers[e] = weights_ctx->dev_buffer; + weight_sizes[e] = weights_ctx->dev_buffer->size; + const uint64_t gate_off = params->gate_byte_offsets[e]; + const uint64_t up_off = params->up_byte_offsets[e]; + const uint64_t down_off = params->down_byte_offsets[e]; + // The real alignment gate: binding a role's sub-buffer at a + // misaligned offset is a Vulkan validity violation, not merely + // slow. gate/up/down are validated separately since nothing + // guarantees they share a phase. + if (align == 0 || gate_off % align != 0 || up_off % align != 0 || + down_off % align != 0) { + return false; } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_f16.f32acc; + if (gate_off > weight_sizes[e] || gate_role_bytes > weight_sizes[e] - gate_off || + up_off > weight_sizes[e] || gate_role_bytes > weight_sizes[e] - up_off || + down_off > weight_sizes[e] || down_role_bytes > weight_sizes[e] - down_off) { + return false; } } - // MMQ - if (src1_type == GGML_TYPE_Q8_1) { - vk_matmul_pipeline pipelines = ctx->device->pipeline_dequant_mul_mat_mat_q8_1[src0_type].f32acc; - - if (pipelines->is_empty()) { - return nullptr; - } - - return pipelines; + const size_t header_words = 9; + const size_t route_words = (size_t) params->n_experts > + std::numeric_limits<size_t>::max() / params->n_tokens + ? 0 : (size_t) params->n_experts * params->n_tokens; + if (route_words == 0 || route_words > std::numeric_limits<size_t>::max() - header_words) { + return false; + } + const size_t total_words = header_words + route_words; + const size_t upload_bytes = total_words * sizeof(uint32_t); + if (upload_bytes / sizeof(uint32_t) != total_words || + params->params_offset > params_ctx->dev_buffer->size || + upload_bytes > params_ctx->dev_buffer->size - params->params_offset) { + return false; } - if (src1_type != GGML_TYPE_F32 && !ctx->device->coopmat2) { - return nullptr; + const VkDeviceSize max_storage_buffer_range = + ctx->device->properties.limits.maxStorageBufferRange; + if ((VkDeviceSize) io_bytes > max_storage_buffer_range || + (VkDeviceSize) upload_bytes > max_storage_buffer_range || + (VkDeviceSize) gate_role_bytes > max_storage_buffer_range || + (VkDeviceSize) down_role_bytes > max_storage_buffer_range) { + return false; } - switch (src0_type) { - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: - break; - default: - return nullptr; - } - - if (ctx->device->coopmat2) { - assert(src1_type == GGML_TYPE_F16); - return prec == GGML_PREC_DEFAULT ? ctx->device->pipeline_dequant_mul_mat_mat_f16[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat_f16[src0_type].f32acc; - } - if (ctx->device->coopmat_support) { - return (ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec == GGML_PREC_DEFAULT) ? ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f32acc; + const bool down_q8 = params->down_type == GGML_TYPE_Q8_0; + vk_pipeline pipeline = down_q8 + ? ctx->device->pipeline_wp_fused_expert_q8_0 + : ctx->device->pipeline_wp_fused_expert_q5_1; + // Batch fast path is now two pipelines (see the struct member comment on + // pipeline_wp_fused_expert_batch_a): phase A has no down-type variant, + // phase B does. + vk_pipeline batch_pipeline_a = ctx->device->pipeline_wp_fused_expert_batch_a; + vk_pipeline batch_pipeline_b = down_q8 + ? ctx->device->pipeline_wp_fused_expert_batch_b_q8_0 + : ctx->device->pipeline_wp_fused_expert_batch_b_q5_1; + if (ctx->device->subgroup_size != 64 || !ctx->device->subgroup_arithmetic || + ctx->device->properties.limits.maxComputeWorkGroupInvocations < 512 || + pipeline == nullptr) { + return false; } - return (ctx->device->fp16 && prec == GGML_PREC_DEFAULT) ? ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f32acc; -} - -static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * ctx, ggml_type a_type, ggml_type b_type, uint32_t num_cols, uint32_t m, uint32_t k) { - VK_LOG_DEBUG("ggml_vk_get_dequantize_mul_mat_vec()"); - GGML_ASSERT(b_type == GGML_TYPE_F32 || b_type == GGML_TYPE_F16 || b_type == GGML_TYPE_Q8_1); - GGML_ASSERT(num_cols >= 1 && num_cols <= ctx->device->mul_mat_vec_max_cols_eff); - if (b_type == GGML_TYPE_Q8_1) { - switch (a_type) { - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_MXFP4: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - break; - default: - return nullptr; + // *** WP_VK_FUSED_EXPERT_BATCH: single-request, whole-GPU fast path. *** + // + // The per-expert loop below launches n_tokens (1-4 at decode) workgroups + // PER EXPERT -- one CU busy, the other 35 idle on this 36-CU part, and + // experts run one after another because each dispatch folds into the + // SAME output range the previous one wrote (a real RAW hazard, hence the + // barrier at the bottom of that loop). wp_fused_expert_batch_a.comp / + // wp_fused_expert_batch_b.comp fix both problems at once: phase A + // computes silu(gate)*up for every selected expert/token/hidden-unit in + // ONE dispatch (grid = n_experts * n_tokens * ceil(n_ff/8), i.e. ~200+ + // workgroups at 9 experts instead of 1), phase B then does the + // down-projection and folds across experts INSIDE one shader invocation + // per output row (grid = n_tokens * ceil(n_embd/8)) instead of across + // dispatches, so there is only one inter-phase barrier total (the + // ggml_vk_sync_buffers call between the two dispatches below), not one + // per expert. Unlike the earlier single-shader design, phase B contains + // no barrier() of its own and no shared-memory staging -- see + // wp_fused_expert_batch_b.comp's top-of-file comment for why that + // mattered (it hung this device via ErrorDeviceLost). + // + // The catch: these shaders bind gate/up/down as ONE whole-arena storage + // buffer per role and compute each expert's byte base in-shader (see + // FUSED_BATCH_MAX_EXPERTS below), so every selected expert must live in + // the SAME VkBuffer. That is NOT guaranteed in general -- Vulkan's + // allocate_slot_arenas() (wp-expert-worker.cpp) caps each arena at + // ggml_backend_buft_get_max_size() and splits any bigger pool across + // several arenas, and a request's selected experts can land in + // different ones -- so this is checked per call, not assumed, and falls + // through to the unchanged per-expert loop whenever it doesn't hold. + constexpr uint32_t FUSED_BATCH_MAX_EXPERTS = 16; + // Must match wp_fused_expert_batch_a.comp/wp_fused_expert_batch_b.comp's + // compile-time MAX_FF / MAX_TOKENS / MAX_EMBD caps -- every shader loop + // is bounded by these IN ADDITION TO its runtime count, and every value + // read from the params buffer that feeds a loop bound is clamped to the + // matching MAX_* again in-shader. FUSED_BATCH_MAX_FF/MAX_EXPERTS also + // size the scratch buffer below (n_experts * n_tokens * n_ff floats), + // which phase B now reads directly (no more shared-memory hidden cache, + // so there is no maxComputeSharedMemorySize gate to check here any + // more). + constexpr size_t FUSED_BATCH_MAX_FF = 256; + constexpr uint32_t FUSED_BATCH_MAX_TOKENS = 8; + constexpr uint32_t FUSED_BATCH_MAX_EMBD = 4096; + // Kill switch for the batch geometry only, independent of the other + // WP_VK_* env reads above: default on (1), set to 0 to force every + // request through the unchanged per-expert dispatch loop below without + // a rebuild, so the two geometries can be A/B'd against each other. + static const bool batch_geometry_enabled = [] { + const char * e = getenv("WP_VK_FUSED_EXPERT_BATCH"); + return e == nullptr || e[0] != '0'; + }(); + bool batch_fast_path = batch_geometry_enabled && + batch_pipeline_a != nullptr && batch_pipeline_b != nullptr && + params->n_experts <= FUSED_BATCH_MAX_EXPERTS && + params->n_tokens <= FUSED_BATCH_MAX_TOKENS && + params->n_ff <= FUSED_BATCH_MAX_FF && + params->n_embd <= FUSED_BATCH_MAX_EMBD && + (VkDeviceSize) weight_sizes[0] <= max_storage_buffer_range; + for (uint32_t e = 1; batch_fast_path && e < params->n_experts; ++e) { + // weight_buffers[e] is a shared_ptr<vk_buffer_struct>; equality here + // compares the pointee, i.e. "is this the same VkBuffer" -- exactly + // the identity the whole-buffer binding needs. + if (weight_buffers[e] != weight_buffers[0]) { + batch_fast_path = false; } } - switch (a_type) { - case GGML_TYPE_F32: - case GGML_TYPE_F16: - case GGML_TYPE_BF16: - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: - break; - default: - return nullptr; - } + std::lock_guard<std::recursive_mutex> guard(ctx->device->mutex); + const auto started = std::chrono::steady_clock::now(); + const bool submit_async = params->submit_async != 0 && !ctx->device->serialize_submissions; + vk_context compute_ctx = ggml_vk_get_compute_ctx(ctx); + uint64_t n_pipeline_dispatches = 0; + uint64_t n_workgroups = 0; + uint64_t bytes_uploaded_actual = 0; - // heuristic to choose workgroup size - uint32_t dmmv_wg = DMMV_WG_SIZE_SUBGROUP; - if ((ctx->device->vendor_id == VK_VENDOR_ID_NVIDIA && ctx->device->architecture != vk_device_architecture::NVIDIA_PRE_TURING) || ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { - // NOTE (2026-08-17): tried adding AMD_GCN here (DMMV_WG_SIZE_LARGE for the - // MoE expert mat-vec) -- MEASURED a REGRESSION on gfx803/RX480: 9.74 -> - // 13.63 ms/req. The larger workgroup's barrier/tree-reduction overhead - // outweighs the extra parallelism on Polaris. GCN stays on SUBGROUP. - // Prefer larger workgroups when M is small, to spread the work out more - // and keep more SMs busy. - // q6_k seems to prefer small workgroup size even for "medium" values of M. - if (a_type == GGML_TYPE_Q6_K) { - if (m < 4096 && k >= 1024) { - dmmv_wg = DMMV_WG_SIZE_LARGE; - } - } else { - if (m <= 8192 && k >= 1024) { - dmmv_wg = DMMV_WG_SIZE_LARGE; - } + if (batch_fast_path) { + // Header layout private to the batch shader (10 words): unlike the + // per-expert path's header, gate/up/down bases travel in the params + // buffer too (as BYTE offsets into the single whole-arena binding), + // since there is no per-expert sub-buffer rebind to carry them + // implicitly. n_embd/n_ff fully determine gate_blocks/down_blocks, + // so those are recomputed in-shader instead of uploaded. + const size_t batch_header_words = 10; + const size_t batch_route_words = route_words; // already validated != 0, no overflow + if (params->n_experts > std::numeric_limits<size_t>::max() - + (batch_header_words + batch_route_words)) { + return false; } - } - - if (b_type == GGML_TYPE_Q8_1) { - if (ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { - dmmv_wg = DMMV_WG_SIZE_SUBGROUP; + const size_t gate_off_base = batch_header_words + batch_route_words; + const size_t up_off_base = gate_off_base + params->n_experts; + const size_t down_off_base = up_off_base + params->n_experts; + const size_t batch_total_words = down_off_base + params->n_experts; + const size_t batch_upload_bytes = batch_total_words * sizeof(uint32_t); + if (batch_upload_bytes / sizeof(uint32_t) != batch_total_words || + params->params_offset > params_ctx->dev_buffer->size || + batch_upload_bytes > params_ctx->dev_buffer->size - params->params_offset || + (VkDeviceSize) batch_upload_bytes > max_storage_buffer_range) { + return false; } - return ctx->device->pipeline_dequant_mul_mat_vec_q8_1_f32[dmmv_wg][a_type][num_cols-1]; - } - return b_type == GGML_TYPE_F32 ? ctx->device->pipeline_dequant_mul_mat_vec_f32_f32[dmmv_wg][a_type][num_cols-1] : ctx->device->pipeline_dequant_mul_mat_vec_f16_f32[dmmv_wg][a_type][num_cols-1]; -} - -static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_id_pipeline(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { - VK_LOG_DEBUG("ggml_vk_get_mul_mat_mat_id_pipeline()"); - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_id_f32; - } - if (src0_type == GGML_TYPE_BF16 && src1_type == GGML_TYPE_BF16) { - return ctx->device->pipeline_matmul_id_bf16; - } - if (prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support)) { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_id_f16_f32.f16acc; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_id_f16.f16acc; - } - } else { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_id_f16_f32.f32acc; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_id_f16.f32acc; + // Per-expert BYTE offsets fit in uint32 without a lo/hi split: the + // whole-arena binding's range is <= max_storage_buffer_range, and + // VkPhysicalDeviceLimits::maxStorageBufferRange is itself a + // uint32_t field in the Vulkan spec, so any offset within that + // binding is representable in 32 bits. + std::vector<uint32_t> batch_upload(batch_total_words, 0); + batch_upload[0] = params->n_tokens; + batch_upload[1] = params->n_embd; + batch_upload[2] = params->n_ff; + batch_upload[3] = params->n_experts; + batch_upload[4] = params->add_previous != 0; + std::memcpy(&batch_upload[5], ¶ms->swiglu_limit, sizeof(float)); + batch_upload[6] = (uint32_t) batch_header_words; // route_weights_base + batch_upload[7] = (uint32_t) gate_off_base; + batch_upload[8] = (uint32_t) up_off_base; + batch_upload[9] = (uint32_t) down_off_base; + for (uint32_t e = 0; e < params->n_experts; ++e) { + for (uint32_t token = 0; token < params->n_tokens; ++token) { + std::memcpy(&batch_upload[batch_header_words + (size_t) e * params->n_tokens + token], + ¶ms->route_weights[(size_t) e * params->n_tokens + token], + sizeof(float)); + } + batch_upload[gate_off_base + e] = (uint32_t) params->gate_byte_offsets[e]; + batch_upload[up_off_base + e] = (uint32_t) params->up_byte_offsets[e]; + batch_upload[down_off_base + e] = (uint32_t) params->down_byte_offsets[e]; } - } - - // MMQ - if (src1_type == GGML_TYPE_Q8_1) { - vk_matmul_pipeline pipelines = ctx->device->pipeline_dequant_mul_mat_mat_id_q8_1[src0_type].f32acc; - if (pipelines->is_empty()) { - return nullptr; + const size_t scratch_floats = + (size_t) params->n_experts * params->n_tokens * params->n_ff; + const size_t scratch_bytes = scratch_floats * sizeof(float); + if (scratch_floats / params->n_tokens / params->n_experts != params->n_ff || + !ggml_vk_ensure_wp_fused_batch_scratch_buffer(ctx->device, scratch_bytes)) { + return false; // scratch OOM: fail the call rather than silently mis-fusing } - return pipelines; - } - - GGML_ASSERT(src1_type == GGML_TYPE_F32 || (ctx->device->coopmat2 && src1_type == GGML_TYPE_F16)); - - switch (src0_type) { - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: - break; - default: - return nullptr; - } + // One descriptor set per pipeline (phase A and phase B are now + // separate vk_pipeline objects with different binding layouts, so + // each needs its own descriptor-set request -- unlike the old + // single-shader design's one request(..., 2) for two dispatches of + // the SAME pipeline). + ggml_pipeline_request_descriptor_sets(ctx, batch_pipeline_a, 1); + ggml_pipeline_request_descriptor_sets(ctx, batch_pipeline_b, 1); + if (!ggml_vk_buffer_write_async(compute_ctx, params_ctx->dev_buffer, + params->params_offset, batch_upload.data(), + batch_upload_bytes, true)) { + return false; + } + compute_ctx->s->buffer->buf.pipelineBarrier( + vk::PipelineStageFlagBits::eTransfer | vk::PipelineStageFlagBits::eHost, + vk::PipelineStageFlagBits::eComputeShader, + {}, + { { vk::AccessFlagBits::eTransferWrite | vk::AccessFlagBits::eHostWrite, + vk::AccessFlagBits::eShaderRead } }, + {}, {}); - vk_matmul_pipeline2& mmp = ctx->device->pipeline_dequant_mul_mat_mat_id[src0_type]; - // XXX TODO 'prec' is not actually allowed in mul_mat_id. - bool prefer_fp16acc = ctx->device->fp16 /*&& prec == GGML_PREC_DEFAULT*/; - bool support_fp16acc = !mmp.f16acc->is_empty(); - bool support_fp32acc = !mmp.f32acc->is_empty(); + const vk_subbuffer d_input{io_ctx->dev_buffer, params->input_offset, io_bytes}; + const vk_subbuffer d_output{io_ctx->dev_buffer, params->output_offset, io_bytes}; + const vk_subbuffer d_params{params_ctx->dev_buffer, params->params_offset, batch_upload_bytes}; + // The whole arena, offset 0 -- every selected expert's role data is + // somewhere inside it; the shader computes each block's byte + // address as (this base) + (per-expert offset, from the params + // buffer) + (local byte offset within the quant block), all at byte + // granularity, so no VkDescriptorBufferInfo offset + // alignment applies here (unlike the per-expert path's sub-buffer + // binds, which must satisfy minStorageBufferOffsetAlignment). + const vk_subbuffer d_weights{weight_buffers[0], 0, (VkDeviceSize) weight_sizes[0]}; + const vk_subbuffer d_scratch{ctx->device->wp_fused_batch_scratch, 0, (VkDeviceSize) scratch_bytes}; - if (support_fp16acc && (prefer_fp16acc || !support_fp32acc)) { - return mmp.f16acc; - } else { - GGML_ASSERT(support_fp32acc); - return mmp.f32acc; - } -} + // Placeholder push constants (see the struct comment): neither phase + // reads this value, it exists only so ggml_vk_dispatch_pipeline's + // push_constant_size assertion has something matching what + // ggml_vk_create_pipeline was told each pipeline's layout uses. + const vk_wp_fused_expert_batch_push_constants pc{0}; -static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_context * ctx, ggml_type a_type, ggml_type b_type, uint32_t m, uint32_t k) { - VK_LOG_DEBUG("ggml_vk_get_dequantize_mul_mat_vec_id()"); - GGML_ASSERT(b_type == GGML_TYPE_F32 || b_type == GGML_TYPE_Q8_1); + // Phase A: gate, up, input, params, scratch (5 bindings, matching + // wp_fused_expert_batch_a.comp's layout(binding=...) declarations + // and the "5" passed to ggml_vk_create_pipeline for + // pipeline_wp_fused_expert_batch_a above). + const uint32_t hgroups = (params->n_ff + 7u) / 8u; + const uint32_t phase_a_wgs = params->n_experts * params->n_tokens * hgroups; + ggml_vk_dispatch_pipeline(ctx, compute_ctx, batch_pipeline_a, + { d_weights, d_weights, d_input, d_params, d_scratch }, + pc, { phase_a_wgs, 1, 1 }); - if (b_type == GGML_TYPE_Q8_1) { - switch (a_type) { - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_MXFP4: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - break; - default: - return nullptr; - } - } + // Phase B's per-row down-projection reads every expert's Phase A + // output for its token -- a real compute-shader-write -> + // compute-shader-read hazard across the whole scratch buffer, same + // shape as (and using the same helper as) the WP_VK_HOST_READ_FASTPATH + // barrier below. This is now the ONLY inter-phase synchronization: + // phase B's shader itself contains no barrier() (see its top-of-file + // comment) and reads hidden_scratch directly instead of staging + // through shared memory, so this single compute-to-compute buffer + // barrier between the two dispatches is both necessary (phase B's + // reads must not race phase A's writes) and sufficient (there is no + // further intra-phase-B ordering requirement to enforce). + ggml_vk_sync_buffers(ctx, compute_ctx); - switch (a_type) { - case GGML_TYPE_F32: - case GGML_TYPE_F16: - case GGML_TYPE_BF16: - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: - break; - default: - return nullptr; - } + // Phase B: down, output, params, scratch (4 bindings, matching + // wp_fused_expert_batch_b.comp's layout and the "4" passed for + // pipeline_wp_fused_expert_batch_b_q5_1/q8_0 above). + const uint32_t rgroups = (params->n_embd + 7u) / 8u; + const uint32_t phase_b_wgs = params->n_tokens * rgroups; + const vk_subbuffer d_down{weight_buffers[0], 0, (VkDeviceSize) weight_sizes[0]}; + ggml_vk_dispatch_pipeline(ctx, compute_ctx, batch_pipeline_b, + { d_down, d_output, d_params, d_scratch }, + pc, { phase_b_wgs, 1, 1 }); - // heuristic to choose workgroup size - uint32_t dmmv_wg = DMMV_WG_SIZE_SUBGROUP; - if ((ctx->device->vendor_id == VK_VENDOR_ID_NVIDIA && ctx->device->architecture != vk_device_architecture::NVIDIA_PRE_TURING) || ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { - // NOTE (2026-08-17): tried adding AMD_GCN here (DMMV_WG_SIZE_LARGE for the - // MoE expert mat-vec) -- MEASURED a REGRESSION on gfx803/RX480: 9.74 -> - // 13.63 ms/req. The larger workgroup's barrier/tree-reduction overhead - // outweighs the extra parallelism on Polaris. GCN stays on SUBGROUP. - // Prefer larger workgroups when M is small, to spread the work out more - // and keep more SMs busy. - // q6_k seems to prefer small workgroup size even for "medium" values of M. - if (a_type == GGML_TYPE_Q6_K) { - if (m < 4096 && k >= 1024) { - dmmv_wg = DMMV_WG_SIZE_LARGE; - } - } else { - if (m <= 8192 && k >= 1024) { - dmmv_wg = DMMV_WG_SIZE_LARGE; + n_pipeline_dispatches = 2; + n_workgroups = (uint64_t) phase_a_wgs + phase_b_wgs; + bytes_uploaded_actual = batch_upload_bytes; + } else { + std::vector<uint32_t> upload(total_words, 0); + upload[0] = params->n_tokens; + upload[1] = params->n_embd; + upload[2] = params->n_ff; + upload[3] = params->n_experts; + upload[4] = (uint32_t) gate_blocks; + upload[5] = (uint32_t) down_blocks; + upload[6] = params->add_previous != 0; + std::memcpy(&upload[7], ¶ms->swiglu_limit, sizeof(float)); + upload[8] = (uint32_t) header_words; + for (uint32_t e = 0; e < params->n_experts; ++e) { + for (uint32_t token = 0; token < params->n_tokens; ++token) { + std::memcpy(&upload[upload[8] + (size_t) e * params->n_tokens + token], + ¶ms->route_weights[(size_t) e * params->n_tokens + token], + sizeof(float)); } } - } - if (b_type == GGML_TYPE_Q8_1) { - if (ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { - dmmv_wg = DMMV_WG_SIZE_SUBGROUP; + ggml_pipeline_request_descriptor_sets(ctx, pipeline, params->n_experts); + if (!ggml_vk_buffer_write_async(compute_ctx, params_ctx->dev_buffer, + params->params_offset, upload.data(), + upload_bytes, true)) { + return false; } - return ctx->device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[dmmv_wg][a_type]; - } - - return ctx->device->pipeline_dequant_mul_mat_vec_id_f32[dmmv_wg][a_type]; -} - -static void * ggml_vk_host_malloc(vk_device& device, size_t size) { - VK_LOG_MEMORY("ggml_vk_host_malloc(" << size << ")"); - vk_buffer buf = ggml_vk_create_buffer(device, size, - {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, - vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); - - if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) { - fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n", - size/1024.0/1024.0); - device->device.freeMemory(buf->device_memory); - device->device.destroyBuffer(buf->buffer); - return nullptr; - } - - std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex); - device->pinned_memory.push_back(std::make_tuple(buf->ptr, size, buf)); - - return buf->ptr; -} - -static void ggml_vk_host_free(vk_device& device, void* ptr) { - if (ptr == nullptr) { - return; - } - VK_LOG_MEMORY("ggml_vk_host_free(" << ptr << ")"); - std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex); - - vk_buffer buf; - size_t index; - for (size_t i = 0; i < device->pinned_memory.size(); i++) { - const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]); - const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]); - if (ptr >= addr && ptr < endr) { - buf = std::get<2>(device->pinned_memory[i]); - index = i; - break; + compute_ctx->s->buffer->buf.pipelineBarrier( + vk::PipelineStageFlagBits::eTransfer | vk::PipelineStageFlagBits::eHost, + vk::PipelineStageFlagBits::eComputeShader, + {}, + { { vk::AccessFlagBits::eTransferWrite | vk::AccessFlagBits::eHostWrite, + vk::AccessFlagBits::eShaderRead } }, + {}, {}); + const vk_subbuffer d_input{io_ctx->dev_buffer, params->input_offset, io_bytes}; + const vk_subbuffer d_output{io_ctx->dev_buffer, params->output_offset, io_bytes}; + const vk_subbuffer d_params{params_ctx->dev_buffer, params->params_offset, upload_bytes}; + for (uint32_t e = 0; e < params->n_experts; ++e) { + // Each role is its own sub-buffer at its own byte offset (see the + // comment on ggml_backend_vk_wp_fused_expert_params) -- the shader + // always indexes local block 0 upward within each, so no per-expert + // block-index math is needed on the device side. + const vk_subbuffer d_gate{weight_buffers[e], + (VkDeviceSize) params->gate_byte_offsets[e], (VkDeviceSize) gate_role_bytes}; + const vk_subbuffer d_up{weight_buffers[e], + (VkDeviceSize) params->up_byte_offsets[e], (VkDeviceSize) gate_role_bytes}; + const vk_subbuffer d_down{weight_buffers[e], + (VkDeviceSize) params->down_byte_offsets[e], (VkDeviceSize) down_role_bytes}; + vk_wp_fused_expert_push_constants pc{e}; + ggml_vk_dispatch_pipeline(ctx, compute_ctx, pipeline, + { d_gate, d_up, d_down, d_input, d_output, d_params }, + pc, { params->n_tokens, 1, 1 }); + if (e + 1 < params->n_experts) { + compute_ctx->s->buffer->buf.pipelineBarrier( + vk::PipelineStageFlagBits::eComputeShader, + vk::PipelineStageFlagBits::eComputeShader, + {}, + { { vk::AccessFlagBits::eShaderWrite, + vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite } }, + {}, {}); + } } + n_pipeline_dispatches = params->n_experts; + n_workgroups = (uint64_t) params->n_experts * params->n_tokens; + bytes_uploaded_actual = upload_bytes; } - if (buf == nullptr) { - fprintf(stderr, "WARNING: failed to free pinned memory: memory not in map\n"); - return; - } - - ggml_vk_destroy_buffer(buf); - device->pinned_memory.erase(device->pinned_memory.begin() + index); -} - -static void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset) { - std::shared_lock<std::shared_mutex> guard(device->pinned_memory_mutex); - buf = nullptr; - buf_offset = 0; - for (size_t i = 0; i < device->pinned_memory.size(); i++) { - const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]); - const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]); - if (ptr >= addr && ptr < endr) { - buf = std::get<2>(device->pinned_memory[i]); - buf_offset = ((const uint8_t *)ptr) - addr; - break; + // *** WP_VK_HOST_READ_FASTPATH -- populate the read-side cache. *** + // + // Mirrors the population half of ggml_backend_vk_graph_compute()'s fastpath + // (see the "submit || last_node" block there): copy this dispatch's output + // range into device->host_read_staging as part of the SAME command buffer + // this dispatch is already recording -- before ctx_end()/submit -- and + // publish host_read_source/offset/size so the next read_result() hits the + // cached staging read instead of a cold device read. Previously this + // function only invalidated the cache (see below) because it writes + // io_ctx->dev_buffer outside graph_compute's own fastpath machinery; that + // made every fused read a real device read (the WP_VK_HOST_READ_FASTPATH + // readback regression). When the fastpath is disabled or ineligible for + // this output, fall through to the same invalidation graph_compute's own + // top-of-function reset performs, so a stale snapshot from a prior graph + // (or a prior fused dispatch whose output didn't qualify) is never served. + { + static const bool host_read_fastpath = [] { + const char * e = getenv("WP_VK_HOST_READ_FASTPATH"); + return e != nullptr && e[0] == '1'; + }(); + static const size_t host_read_fastpath_max = [] { + const char * e = getenv("WP_VK_HOST_READ_FASTPATH_MAX_BYTES"); + return (e != nullptr && e[0] != '\0') ? (size_t) strtoull(e, nullptr, 10) : (size_t) 262144; + }(); + const bool output_host_visible = static_cast<bool>( + io_ctx->dev_buffer->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible); + const bool output_host_cached = static_cast<bool>( + io_ctx->dev_buffer->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached); + if (host_read_fastpath && host_read_fastpath_max != 0 && + output_host_visible && !output_host_cached && io_bytes <= host_read_fastpath_max && + ggml_vk_ensure_host_read_staging_buffer(ctx->device, host_read_fastpath_max)) { + ggml_vk_sync_buffers(ctx, compute_ctx); + compute_ctx->s->buffer->buf.copyBuffer(io_ctx->dev_buffer->buffer, + ctx->device->host_read_staging->buffer, + { { params->output_offset, 0, io_bytes } }); + ctx->device->host_read_source = io_ctx->dev_buffer; + ctx->device->host_read_source_offset = params->output_offset; + ctx->device->host_read_source_size = io_bytes; + } else { + ctx->device->host_read_source.reset(); + ctx->device->host_read_source_offset = 0; + ctx->device->host_read_source_size = 0; } } -} -static vk_subbuffer ggml_vk_tensor_subbuffer( - const ggml_backend_vk_context * ctx, const ggml_tensor * tensor, bool allow_misalign = false) { - - vk_buffer buffer = nullptr; - size_t offset = 0; - if (ctx->device->uma) { - ggml_vk_host_get(ctx->device, tensor->data, buffer, offset); + vk_command_buffer * cmd_buf = compute_ctx->s != nullptr ? compute_ctx->s->buffer : nullptr; + compute_ctx->submitted_cmd_buf = cmd_buf; + ggml_vk_ctx_end(compute_ctx); + for (auto & copy : compute_ctx->in_memcpys) { + std::memcpy(copy.dst, copy.src, copy.n); } - if (!buffer) { - auto buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context; - buffer = buf_ctx->dev_buffer; - offset = vk_tensor_offset(tensor) + tensor->view_offs; + compute_ctx->in_memcpys.clear(); + if (submit_async) { + // Attach ctx->fence directly to this dispatch's own submission instead + // of a null-fence submit followed by a second, empty, fence-only + // vkQueueSubmit inside ggml_vk_synchronize()/ggml_backend_vk_synchronize(). + // This is the same WP_VK_FENCE_ON_LAST_SUBMIT trick already applied to + // the normal graph-compute path (see the comment at its call site in + // ggml_vk_compute_forward): a fence signals only after ITS submission + // AND every earlier submission on the (in-order) queue complete, which + // is exactly the guarantee the separate empty submit provided. Every + // fused-expert call is already a single, self-contained submission (one + // command buffer, nothing else queued on this context before the + // caller's synchronize), so the fence can always ride the real submit + // here -- no env-var gate needed, unlike the tiny-graph case upstream. + // Same command buffer, same order, same fence semantics -- one fewer + // queue submission and one fewer wait per request. + // + // Maintain the fence_submitted invariant: never leave an unconsumed + // fence signal behind while submitting more work onto the same fence + // (mirrors the "drain first" check in ggml_vk_compute_forward). + if (ctx->fence_submitted) { + ggml_vk_wait_for_fence(ctx); + ggml_vk_queue_mark_waited(*ctx->device->compute_queue->handle, ctx->fence_epoch); + ctx->fence_submitted = false; + ctx->submit_pending = false; + } + ggml_vk_submit(compute_ctx, ctx->fence); + ctx->fence_submitted = true; + ctx->fence_epoch = ggml_vk_queue_epoch(*ctx->device->compute_queue->handle); + ctx->submit_pending = true; + } else { + ggml_vk_submit(compute_ctx, ctx->fence); + const uint64_t epoch = ggml_vk_queue_epoch(*ctx->device->compute_queue->handle); + ggml_vk_wait_for_fence(ctx); + ggml_vk_queue_mark_waited(*ctx->device->compute_queue->handle, epoch); + ctx->device->device.resetFences({ ctx->fence }); + if (cmd_buf != nullptr) { + cmd_buf->in_use = false; + cmd_buf->buf.reset(); + compute_ctx->submitted_cmd_buf = nullptr; + } + ggml_vk_queue_command_pools_cleanup(ctx->device); + ctx->compute_ctx.reset(); } - GGML_ASSERT(buffer != nullptr); - - size_t size = ggml_nbytes(tensor); - - size_t misalign_bytes = offset & (ctx->device->properties.limits.minStorageBufferOffsetAlignment - 1); - // The shader must support misaligned offsets when indexing into the buffer - GGML_ASSERT(allow_misalign || misalign_bytes == 0); - offset &= ~misalign_bytes; - size += misalign_bytes; - - return vk_subbuffer{buffer, offset, size, tensor, allow_misalign}; + // WP_VK_HOST_READ_FASTPATH cache: populated (or, when ineligible, + // invalidated) above, inside the same command buffer this dispatch + // recorded, before it was ended and submitted. + stats->ns_dispatch = (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>( + std::chrono::steady_clock::now() - started).count(); + stats->n_pipeline_dispatches = n_pipeline_dispatches; + stats->n_fence_waits = submit_async ? 0 : 1; + stats->bytes_uploaded = bytes_uploaded_actual; + stats->n_workgroups = n_workgroups; + return true; } -static vk_subbuffer ggml_vk_mark_dynamic_subbuffer( - const ggml_backend_vk_context * ctx, vk_subbuffer buffer, const ggml_tensor * tensor) { - if (ctx->recording_plan != nullptr) { - buffer.tensor = tensor; +// Take-and-clear, so a publication can never be applied to two nodes. +static bool ggml_vk_wp_take_expert_offsets(vk_wp_expert_offsets & out) { + if (!tls_wp_expert_offsets.valid) { + return false; } - return buffer; + out = tls_wp_expert_offsets; + tls_wp_expert_offsets.valid = false; + return true; } -static void ggml_vk_graph_plan_update_descriptors( - ggml_backend_vk_context * ctx, ggml_backend_vk_graph_plan * plan) { - bool has_dynamic_binding = false; - for (const auto & update : plan->descriptor_updates) { - has_dynamic_binding = has_dynamic_binding || std::any_of( - update.sources.begin(), update.sources.end(), - [](const vk_subbuffer & source) { return source.tensor != nullptr; }); - } - if (!has_dynamic_binding) { - return; - } +// Upload one node's per-expert offset array and return the subbuffer to bind. +// +// Each node gets its OWN slice. The dispatches recorded here do not run until +// the graph's command buffer is submitted, so a single shared region written +// eagerly per node would leave every dispatch reading the LAST node's offsets. +// (CUDA sidesteps this because its pointer upload is a stream-ordered async +// memcpy interleaved with the kernels.) The cursor is reset per graph in +// ggml_backend_vk_graph_compute. +static vk_subbuffer ggml_vk_wp_upload_expert_offsets(ggml_backend_vk_context * ctx, + const std::vector<uint32_t> & offsets) { + const size_t nbytes = offsets.size() * sizeof(uint32_t); + const size_t align = std::max<size_t>( + ctx->device->properties.limits.minStorageBufferOffsetAlignment, 4); + const size_t stride = (nbytes + align - 1) / align * align; - std::vector<std::vector<vk::DescriptorBufferInfo>> infos; - std::vector<vk::WriteDescriptorSet> writes; - infos.reserve(plan->descriptor_updates.size()); - writes.reserve(plan->descriptor_updates.size()); - for (const auto & update : plan->descriptor_updates) { - infos.emplace_back(); - infos.back().reserve(update.sources.size()); - for (const vk_subbuffer & source : update.sources) { - const vk_subbuffer current = source.tensor != nullptr - ? ggml_vk_tensor_subbuffer(ctx, source.tensor, source.allow_misalign) - : source; - infos.back().push_back(current); + if (ctx->prealloc_wp_expert_off == nullptr || + ctx->prealloc_wp_expert_off->size < ctx->wp_expert_off_cursor + stride) { + const size_t grow = std::max<size_t>( + ctx->wp_expert_off_cursor + stride * 64, 64 * 1024); + vk_buffer old = ctx->prealloc_wp_expert_off; + ctx->prealloc_wp_expert_off = ggml_vk_create_buffer_device(ctx->device, grow); + if (old != nullptr) { + // Anything already recorded still points into `old`; keep it alive + // until the graph completes rather than freeing here. + ctx->wp_expert_off_retired.push_back(old); } - writes.push_back({ - update.descriptor_set, 0, 0, (uint32_t) infos.back().size(), - vk::DescriptorType::eStorageBuffer, nullptr, infos.back().data() - }); + ctx->wp_expert_off_cursor = 0; } - ctx->device->device.updateDescriptorSets(writes, {}); + + const size_t slice = ctx->wp_expert_off_cursor; + ggml_vk_buffer_write(ctx->prealloc_wp_expert_off, slice, offsets.data(), nbytes); + ctx->wp_expert_off_cursor += stride; + return vk_subbuffer{ ctx->prealloc_wp_expert_off, slice, nbytes }; } -// Get a command buffer from pool. Create a new one if no reusable buffer is available -static vk_command_buffer* ggml_vk_get_or_create_cmd_buffer(vk_device& device, vk_command_pool& pool) { - for (auto& cmd_buffer : pool.cmd_buffers) { - if (!cmd_buffer.in_use) { - cmd_buffer.use_counter++; - cmd_buffer.in_use = true; - return &cmd_buffer; - } +bool ggml_backend_vk_wp_event_query(void * event) { + auto * wp_event = (ggml_vk_wp_event *) event; + if (wp_event == nullptr) { + return false; } - return ggml_vk_create_cmd_buffer(device, pool); + return wp_event->device->device.getFenceStatus(wp_event->fence) == vk::Result::eSuccess; } -static vk_submission ggml_vk_begin_submission(vk_device& device, vk_command_pool& p, bool one_time = true) { - vk_submission s; - s.buffer = ggml_vk_get_or_create_cmd_buffer(device, p); - if (one_time) { - s.buffer->buf.begin({ vk::CommandBufferUsageFlagBits::eOneTimeSubmit }); - } else { - s.buffer->buf.begin({ vk::CommandBufferUsageFlags{} }); +bool ggml_backend_vk_wp_event_wait(void * event) { + auto * wp_event = (ggml_vk_wp_event *) event; + if (wp_event == nullptr) { + return false; } - - return s; + std::lock_guard<std::recursive_mutex> guard(wp_event->device->mutex); + VK_CHECK(wp_event->device->device.waitForFences({ wp_event->fence }, true, UINT64_MAX), "wp stage_in waitForFences", wp_event->device); + ggml_vk_queue_command_pools_cleanup(wp_event->device); + return true; } -template <typename T> size_t push_constant_size(const T &t) { - static_assert(std::is_class<T>::value, "T must be a struct/class"); - GGML_UNUSED(t); - return sizeof(T); -} -template <typename T> size_t push_constant_size(const std::vector<T> &t) { - GGML_UNUSED(t); - return sizeof(T) * t.size(); -} -template <typename T, uint32_t N> size_t push_constant_size(const std::array<T, N> &t) { - GGML_UNUSED(t); - return sizeof(T) * N; +void ggml_backend_vk_wp_event_free(void * event) { + auto * wp_event = (ggml_vk_wp_event *) event; + if (wp_event == nullptr) { + return; + } + ggml_backend_vk_wp_event_wait(wp_event); + wp_event->device->device.destroyFence(wp_event->fence); + delete wp_event; } -template <typename T> const T *push_constant_data(const T &t) { - static_assert(std::is_class<T>::value, "T must be a struct/class"); - return &t; -} -template <typename T> const T *push_constant_data(const std::vector<T> &t) { - return t.data(); -} -template <typename T, uint32_t N> const T *push_constant_data(const std::array<T, N> &t) { - return t.data(); -} +static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, uint32_t n, uint32_t k, bool disable_split_k, const vk_pipeline& pipeline) { + VK_LOG_DEBUG("ggml_vk_guess_split_k(" << m << ", " << n << ", " << k << ", " << disable_split_k << ")"); -template <typename T> -static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list<vk_subbuffer> const& descriptor_buffers, const T &push_constants, std::array<uint32_t, 3> elements) { - const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]); - const uint32_t wg1 = CEIL_DIV(elements[1], pipeline->wg_denoms[1]); - const uint32_t wg2 = CEIL_DIV(elements[2], pipeline->wg_denoms[2]); - VK_LOG_DEBUG("ggml_vk_dispatch_pipeline(" << pipeline->name << ", {"; - for (auto& buffer : descriptor_buffers) { - std::cerr << "(" << buffer.buffer->buffer << ", " << buffer.offset << ", " << buffer.size << "), "; + if (disable_split_k) { + return 1; } - std::cerr << "}, (" << wg0 << "," << wg1 << "," << wg2 << "))"); - GGML_ASSERT(wg0 <= ctx->device->properties.limits.maxComputeWorkGroupCount[0] && - wg1 <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && - wg2 <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]); - std::vector<vk::DescriptorSet> & descriptor_sets = ggml_vk_descriptor_sets(ctx); - uint32_t & descriptor_set_idx = ggml_vk_descriptor_set_idx(ctx); - GGML_ASSERT(descriptor_set_idx < descriptor_sets.size()); - GGML_ASSERT(descriptor_buffers.size() <= MAX_PARAMETER_COUNT); - GGML_ASSERT(pipeline->parameter_count == descriptor_buffers.size()); - GGML_ASSERT(pipeline->push_constant_size == push_constant_size(push_constants)); - - vk::DescriptorSet& descriptor_set = descriptor_sets[descriptor_set_idx++]; - std::array<vk::DescriptorBufferInfo, MAX_PARAMETER_COUNT> & descriptor_buffer_infos = - ctx->descriptor_buffer_infos; - size_t descriptor_buffer_count = 0; - for (const vk_subbuffer & buffer : descriptor_buffers) { - descriptor_buffer_infos[descriptor_buffer_count++] = buffer; - } - vk::WriteDescriptorSet write_descriptor_set{ - descriptor_set, 0, 0, pipeline->parameter_count, - vk::DescriptorType::eStorageBuffer, nullptr, descriptor_buffer_infos.data() - }; - ctx->device->device.updateDescriptorSets({ write_descriptor_set }, {}); - if (ctx->recording_plan != nullptr) { - ggml_backend_vk_graph_plan::descriptor_update update; - update.descriptor_set = descriptor_set; - update.sources.assign(descriptor_buffers.begin(), descriptor_buffers.end()); - for (vk_subbuffer & source : update.sources) { - if (source.tensor != nullptr) { - source.buffer.reset(); + uint32_t split_k = 1; + if (ctx->device->shader_core_count != 0 && n >= pipeline->wg_denoms[1]) { + // If k is 'large' and the SMs will fill less than halfway, use split_k. + uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]); + uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]); + + if (k >= 2048) { + if (m_tiles * n_tiles <= ctx->device->shader_core_count / 2) { + split_k = ctx->device->shader_core_count / (m_tiles * n_tiles); + } else if (m_tiles * n_tiles <= ctx->device->shader_core_count * 2 / 3) { + split_k = 3; } - } - ctx->recording_plan->descriptor_updates.push_back(std::move(update)); - } + // Cap the split at 8x. Unless k is huge this is a lot of overhead. + split_k = std::min(split_k, 8u); - subctx->s->buffer->buf.pushConstants(pipeline->layout, vk::ShaderStageFlagBits::eCompute, 0, push_constant_size(push_constants), push_constant_data(push_constants)); - subctx->s->buffer->buf.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline->pipeline); - subctx->s->buffer->buf.bindDescriptorSets(vk::PipelineBindPoint::eCompute, - pipeline->layout, - 0, - { descriptor_set }, - {}); - subctx->s->buffer->buf.dispatch(wg0, wg1, wg2); -} - -static void ggml_vk_flush_cpy_regions(vk_context& ctx) { - if (ctx == nullptr || ctx->cpy_regions.empty()) { - return; - } - GGML_ASSERT(ctx->s != nullptr); - GGML_ASSERT(ctx->cpy_src != nullptr && ctx->cpy_dst != nullptr); - ctx->s->buffer->buf.copyBuffer(ctx->cpy_src->buffer, ctx->cpy_dst->buffer, ctx->cpy_regions); - ctx->cpy_regions.clear(); - ctx->cpy_src.reset(); - ctx->cpy_dst.reset(); -} - -static void ggml_vk_ctx_end(vk_context& ctx) { - VK_LOG_DEBUG("ggml_vk_ctx_end(" << ctx << ", " << ctx->seqs.size() << ")"); - if (ctx->s == nullptr) { - return; + // ggml_vk_matmul will align the splits to be a multiple of 256. + // If this rounded up size would cause the last split to be empty, + // then reduce the split count. + while (true) { + if (split_k == 1) { + break; + } + uint32_t k_split = CEIL_DIV(k, split_k); + k_split = ROUNDUP_POW2(k_split, 256); + if (k_split * (split_k - 1) < k) { + break; + } + split_k--; + } + } } - ggml_vk_flush_cpy_regions(ctx); - ctx->s->buffer->buf.end(); - ctx->s = nullptr; + return split_k; } -static void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx, bool one_time = true) { - VK_LOG_DEBUG("ggml_vk_ctx_begin(" << device->name << ")"); - if (subctx->s != nullptr) { - ggml_vk_ctx_end(subctx); - } +void ggml_vk_matmul( + ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, + vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, + uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, + uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, + uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, + uint32_t padded_n) { + VK_LOG_DEBUG("ggml_vk_matmul(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << (split_k_buffer.buffer != nullptr ? split_k_buffer.buffer->buffer : VK_NULL_HANDLE) << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ", padded_n: " << padded_n << ")"); + if (split_k == 1) { + ggml_pipeline_request_descriptor_sets(ctx, pipeline, CEIL_DIV(batch, ctx->device->properties.limits.maxComputeWorkGroupCount[2])); - subctx->seqs.push_back({ ggml_vk_begin_submission(device, *subctx->p, one_time) }); - subctx->s = subctx->seqs[subctx->seqs.size() - 1].data(); -} + uint32_t base_work_group_z = 0; + while (base_work_group_z < batch) { + uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]); -static vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) { - vk_context result; - if (!ctx->compute_ctx.expired()) { - result = ctx->compute_ctx.lock(); - } else { - if (ctx->recording_plan != nullptr) { - result = ggml_vk_create_temporary_context(ctx->recording_plan->command_pool); - ctx->gc.contexts.emplace_back(result); - } else { - result = ggml_vk_create_context(ctx, ctx->compute_cmd_pool); + const vk_mat_mat_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k, ne02, ne12, broadcast2, broadcast3, padded_n }; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d }, pc, { m, n, groups_z }); + base_work_group_z += groups_z; } - - ctx->compute_ctx = result; - ggml_vk_ctx_begin(ctx->device, result, ctx->recording_plan == nullptr); + return; } - if (ctx->recording_plan == nullptr && ctx->device->async_use_transfer_queue && - ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) { - result->s->wait_semaphores.push_back(ctx->transfer_semaphore); - ctx->transfer_semaphore_last_submitted = ctx->transfer_semaphore.value; + if (ctx->prealloc_split_k_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); } - return result; -} - -static vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx) { - vk_context result; - if (!ctx->transfer_ctx.expired()) { - result = ctx->transfer_ctx.lock(); - } else { - result = ggml_vk_create_context(ctx, ctx->transfer_cmd_pool); - - ctx->transfer_ctx = result; - ggml_vk_ctx_begin(ctx->device, result); - } + GGML_ASSERT(batch_stride_d == m * n); - return result; -} + // Round the split size up to a multiple of 256 (k-quant alignment) + uint32_t k_split = CEIL_DIV(k, split_k); + k_split = ROUNDUP_POW2(k_split, 256); -// Submit any pending transfer queue work and signal the transfer semaphore. -// The next compute context created via ggml_vk_get_compute_ctx will wait on this semaphore. -// Returns true if work was submitted. -static bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx) { - if (!ctx->device->async_use_transfer_queue || ctx->transfer_ctx.expired()) { - return false; - } + ggml_pipeline_request_descriptor_sets(ctx, pipeline, CEIL_DIV(batch, ctx->device->properties.limits.maxComputeWorkGroupCount[2])); - vk_context cpy_ctx = ctx->transfer_ctx.lock(); - ggml_vk_ctx_end(cpy_ctx); + uint32_t base_work_group_z = 0; + while (base_work_group_z < batch) { + uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]); - for (auto& cpy : cpy_ctx->in_memcpys) { - memcpy(cpy.dst, cpy.src, cpy.n); + const vk_mat_mat_push_constants pc1 = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k_split, ne02, ne12, broadcast2, broadcast3, padded_n }; + // Make sure enough workgroups get assigned for split k to work + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer }, pc1, { (CEIL_DIV(m, pipeline->wg_denoms[0]) * pipeline->wg_denoms[0]) * split_k, n, groups_z }); + base_work_group_z += groups_z; } - - ctx->transfer_semaphore.value++; - cpy_ctx->seqs.back().back().signal_semaphores.push_back(ctx->transfer_semaphore); - - ggml_vk_submit(cpy_ctx, {}); - ctx->transfer_ctx.reset(); - return true; + ggml_vk_sync_buffers(ctx, subctx); + const std::array<uint32_t, 2> pc2 = { (uint32_t)(m * n * batch), split_k }; + ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_matmul_split_k_reduce, { split_k_buffer, d }, pc2, { m * n * batch, 1, 1 }); + ctx->prealloc_split_k_need_sync = true; } -static bool ggml_vk_submit_graph_plan( - ggml_backend_vk_context * ctx, ggml_backend_vk_graph_plan * plan) { - if (plan == nullptr || plan->owner != ctx || plan->unsupported || - plan->contexts.empty()) { - return false; - } - - std::vector<vk::CommandBuffer> command_buffers; - for (const vk_context & subctx : plan->contexts) { - if (subctx == nullptr || subctx->seqs.size() != 1) { - return false; - } - for (const vk_submission & submission : subctx->seqs.front()) { - if (submission.buffer == nullptr || - !submission.wait_semaphores.empty() || - !submission.signal_semaphores.empty()) { - return false; - } - command_buffers.push_back(submission.buffer->buf); - } - } - if (command_buffers.empty()) { - return false; +static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { + if (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16) return false; + if (src1_type == GGML_TYPE_Q8_1) return false; + if (src0_type == GGML_TYPE_F16) { + return prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support); } - - vk::Semaphore wait_semaphore; - uint64_t wait_value = 0; - vk::PipelineStageFlags wait_stage; - if (ctx->device->async_use_transfer_queue && - ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) { - wait_semaphore = ctx->transfer_semaphore.s; - wait_value = ctx->transfer_semaphore.value; - wait_stage = ctx->device->compute_queue->stage_flags; - } - - std::vector<vk::SubmitInfo> submit_infos(command_buffers.size()); - for (size_t i = 0; i < command_buffers.size(); ++i) { - submit_infos[i] = { - 0, nullptr, nullptr, - 1, &command_buffers[i], - 0, nullptr, - }; + // quant types + if (ctx->device->coopmat2) { + return prec == GGML_PREC_DEFAULT; } - - vk::TimelineSemaphoreSubmitInfo timeline_info{}; - if (wait_semaphore) { - timeline_info = { 1, &wait_value, 0, nullptr }; - submit_infos.front() = { - 1, &wait_semaphore, &wait_stage, - 1, &command_buffers.front(), - 0, nullptr, - }; - submit_infos.front().setPNext(&timeline_info); - ctx->transfer_semaphore_last_submitted = wait_value; + if (ctx->device->coopmat_support) { + return ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec == GGML_PREC_DEFAULT; } - - ctx->device->compute_queue->handle->submit(submit_infos, {}); - ctx->submit_pending = true; - return true; + return ctx->device->fp16 && prec == GGML_PREC_DEFAULT; } -static size_t ggml_vk_align_size(size_t width, size_t align) { - VK_LOG_DEBUG("ggml_vk_align_size(" << width << ", " << align << ")"); - return CEIL_DIV(width, align) * align; +static const std::vector<vk_matmul_pipeline_pair>* ggml_vk_get_mul_mat_mat_pipeline_map( + ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec, bool mul_mat_id = false) { + bool f16acc = ggml_vk_get_mul_mat_mat_f16acc(ctx, src0_type, src1_type, prec); + vk_matmul_pipeline_key key{src0_type, src1_type, mul_mat_id, f16acc}; + auto it = ctx->device->pipeline_matmul.find(key); + if (it == ctx->device->pipeline_matmul.end() || it->second.empty()) { + // Try without f16acc + if (f16acc) { + key.f16acc = false; + it = ctx->device->pipeline_matmul.find(key); + if (it != ctx->device->pipeline_matmul.end() && !it->second.empty()) return &it->second; + } + return nullptr; + } + return &it->second; } -static void deferred_memcpy(void * dst, const void * src, size_t size, std::vector<vk_staging_memcpy>* memcpys = nullptr) { - if (memcpys == nullptr) { - memcpy(dst, src, size); - } else { - memcpys->emplace_back(dst, src, size); - } +static vk_pipeline ggml_vk_guess_matmul_pipeline_map(ggml_backend_vk_context * ctx, + const std::vector<vk_matmul_pipeline_pair>& configs, + uint32_t m, uint32_t n, bool aligned, bool mul_mat_id) { + auto& selector = mul_mat_id ? ctx->device->matmul_id_tile_selector : ctx->device->matmul_tile_selector; + uint32_t idx = selector(m, n, 0, ctx->device->shader_core_count, configs); + if (idx >= configs.size()) idx = (uint32_t)configs.size() - 1; + return (aligned && configs[idx].aligned) ? configs[idx].aligned : configs[idx].unaligned; } -static void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_staging_memset>* memsets = nullptr) { - if (memsets == nullptr) { - memset(dst, val, size); - } else { - memsets->emplace_back(dst, val, size); - } +static uint32_t ggml_vk_guess_matmul_pipeline_align_map(ggml_backend_vk_context * ctx, + const std::vector<vk_matmul_pipeline_pair>& configs, + uint32_t m, uint32_t n, bool mul_mat_id) { + auto& selector = mul_mat_id ? ctx->device->matmul_id_tile_selector : ctx->device->matmul_tile_selector; + uint32_t idx = selector(m, n, 0, ctx->device->shader_core_count, configs); + if (idx >= configs.size()) idx = (uint32_t)configs.size() - 1; + return configs[idx].align; } -static void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size) { - if (device->sync_staging == nullptr || device->sync_staging->size < size) { - VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")"); - ggml_vk_destroy_buffer(device->sync_staging); - device->sync_staging = ggml_vk_create_buffer_check(device, size, - vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, - vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); - } +static void ggml_vk_matmul_id( + ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, + vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& ids, const vk_subbuffer & expert_count_buf, + const vk_subbuffer & wp_off_buf, + uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, + uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, + uint32_t n_as, uint32_t nei0, uint32_t nei1, uint32_t nbi1, uint32_t ne11, + bool hoist_row_ids, uint32_t paged) { + VK_LOG_DEBUG("ggml_vk_matmul_id(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), ids: (" << ids.buffer->buffer << ", " << ids.offset << ", " << ids.size << "), expert_count: (" << expert_count_buf.buffer->buffer << ", " << expert_count_buf.offset << ", " << expert_count_buf.size << "), " << + "m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", " << + "batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", " << + "n_as: " << n_as << ", nei0: " << nei0 << ", nei1: " << nei1 << ", nbi1: " << nbi1 << ", ne11: " << ne11 << ")"); + const vk_mat_mat_id_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, + nei0, nei1, nbi1, ne11, n_as, uint32_t(hoist_row_ids), paged }; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf, wp_off_buf }, pc, { m, nei1, n_as }); } -static void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size) { - if (ctx->sync_staging == nullptr || ctx->sync_staging->size < size) { - VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")"); - ggml_vk_destroy_buffer(ctx->sync_staging); - ctx->sync_staging = ggml_vk_create_buffer_check(ctx->device, size, - vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, - vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); - } +bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) { + return + tensor->nb[0] == ggml_type_size(tensor->type) && + tensor->nb[1] == (tensor->nb[0]*tensor->ne[0])/ggml_blck_size(tensor->type) && + (tensor->ne[3] == 1 || tensor->nb[3] == tensor->nb[2]*tensor->ne[2]); } -static bool ggml_vk_ensure_host_read_staging_buffer(vk_device& device, size_t size) { - if (device->host_read_staging != nullptr) { - return device->host_read_staging->size >= size && - (device->host_read_staging->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached); - } +vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to) { - try { - device->host_read_staging = ggml_vk_create_buffer(device, size, - {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached}); - } catch (const vk::SystemError&) { - return false; - } + // Choose "contiguous copy" shader if src/dst are contiguous + bool contig = ggml_is_contiguous(src) && (!dst || ggml_is_contiguous(dst)); - return device->host_read_staging != nullptr && - (device->host_read_staging->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached); -} + // Use optimized "transpose" shader if src dim1 is the innermost dimension. + bool transpose = dst && src->nb[1] == ggml_type_size(to) && ggml_are_same_shape(dst, src); -// Device-local (no host-visible requirement -- only Phase A writes it and -// only Phase B reads it, both on-device, in the same command buffer) growable -// scratch buffer for the fused-expert batch fast path. Returns false (leaving -// the fast path unusable for this call) only on an actual allocation failure; -// callers must fall back to the per-expert dispatch loop in that case. -static bool ggml_vk_ensure_wp_fused_batch_scratch_buffer(vk_device& device, size_t size) { - if (device->wp_fused_batch_scratch != nullptr && device->wp_fused_batch_scratch->size >= size) { - return true; - } - try { - ggml_vk_destroy_buffer(device->wp_fused_batch_scratch); - device->wp_fused_batch_scratch = ggml_vk_create_buffer_check( - device, size, vk::MemoryPropertyFlagBits::eDeviceLocal); - } catch (const vk::SystemError&) { - return false; + if (transpose && src->type == to) { + if (ggml_type_size(to) == 4) { + return ctx->device->pipeline_cpy_transpose_32; + } else if (ggml_type_size(to) == 2) { + return ctx->device->pipeline_cpy_transpose_16; + } } - return device->wp_fused_batch_scratch != nullptr; -} -static void ggml_vk_buffer_write_nc_async(ggml_backend_vk_context * ctx, vk_context& subctx, vk_buffer& dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) { - VK_LOG_DEBUG("ggml_vk_buffer_write_nc_async(" << tensor << ")"); - GGML_ASSERT(!ggml_is_contiguous(tensor)); - // Buffer is already mapped - if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { - std::cerr << "ggml_vulkan: buffer_write_nc_async dst buffer is host_visible. Use synchronous write." << std::endl; - GGML_ABORT("fatal error"); - } - // Check if src is pinned memory - vk_buffer buf = nullptr; - size_t buf_offset = 0; - ggml_vk_host_get(ctx->device, tensor->data, buf, buf_offset); - - const uint64_t ne0 = tensor->ne[0]; - const uint64_t ne1 = tensor->ne[1]; - const uint64_t ne2 = tensor->ne[2]; - const uint64_t ne3 = tensor->ne[3]; - const uint64_t nb0 = tensor->nb[0]; - const uint64_t nb1 = tensor->nb[1]; - const uint64_t nb2 = tensor->nb[2]; - const uint64_t nb3 = tensor->nb[3]; - const ggml_type type = tensor->type; - const uint64_t ts = ggml_type_size(type); - const uint64_t bs = ggml_blck_size(type); - - const uint64_t dstnb0 = ts; - const uint64_t dstnb1 = dstnb0*(ne0/bs); - const uint64_t dstnb2 = dstnb1*ne1; - const uint64_t dstnb3 = dstnb2*ne2; - - const uint64_t ne = ggml_nelements(tensor); - - if (buf != nullptr) { - // Memory is pinned, use as staging buffer - std::vector<vk::BufferCopy> slices; - - for (uint64_t i3 = 0; i3 < ne3; i3++) { - for (uint64_t i2 = 0; i2 < ne2; i2++) { - // Find longest contiguous slice - if (ne1*nb1 == dstnb2) { - slices.push_back({ buf_offset + i3*nb3 + i2*nb2, offset + i3*dstnb3 + i2*dstnb2, dstnb2 }); - } else { - for (uint64_t i1 = 0; i1 < ne1; i1++) { - if (ne0*nb0/bs == dstnb1) { - slices.push_back({ buf_offset + i3*nb3 + i2*nb2 + i1*nb1, offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, dstnb1 }); - } else { - const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1; - const uint64_t d_off = offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1; - for (uint64_t i0 = 0; i0 < ne0; i0++) { - slices.push_back({ s_off + i0*nb0, d_off + i0*dstnb0, dstnb0 }); - } - } - } - } - } - } + // Same, for a 0<->2 swap: src dim2 is the innermost dimension. + bool transpose02 = dst && !contig && src->nb[2] == ggml_type_size(to) && + ggml_is_contiguous(dst) && ggml_are_same_shape(dst, src); - ggml_vk_sync_buffers(ctx, subctx); - subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices); - return; + if (transpose02 && src->type == to) { + if (ggml_type_size(to) == 4) { + return ctx->device->pipeline_cpy_transpose_02_32; + } else if (ggml_type_size(to) == 2) { + return ctx->device->pipeline_cpy_transpose_02_16; + } } - if (!sync_staging) { - GGML_ABORT("Asynchronous write to non-pinned memory not supported"); - } - - // Staging buffer required - vk_buffer& staging = ctx->device->sync_staging; - const uint64_t copy_size = ts*ne/bs; - ggml_vk_ensure_sync_staging_buffer(ctx->device, copy_size); - VkBufferCopy buf_copy{ 0, offset, copy_size }; - - ggml_vk_sync_buffers(ctx, subctx); - vkCmdCopyBuffer(subctx->s->buffer->buf, (VkBuffer)staging->buffer, (VkBuffer)dst->buffer, 1, &buf_copy); - - for (uint64_t i3 = 0; i3 < ne3; i3++) { - for (uint64_t i2 = 0; i2 < ne2; i2++) { - // Find longest contiguous slice - if (ne1*nb1 == dstnb2) { - deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2, dstnb2, &subctx->in_memcpys); - } else { - for (uint64_t i1 = 0; i1 < ne1; i1++) { - if (ne0*nb0/bs == dstnb1) { - deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2 + i1*nb1, dstnb1, &subctx->in_memcpys); - } else { - const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1; - const uint64_t d_off = i3*dstnb3 + i2*dstnb2 + i1*dstnb1; - for (uint64_t i0 = 0; i0 < ne0; i0++) { - deferred_memcpy((uint8_t *)staging->ptr + d_off + i0*dstnb0, (const uint8_t *) tensor->data + s_off + i0*nb0, dstnb0, &subctx->in_memcpys); - } - } - } - } + if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_F32) { + if (contig) { + return ctx->device->pipeline_contig_cpy_f32_f32; + } else { + return ctx->device->pipeline_cpy_f32_f32; } } -} - -static bool ggml_vk_buffer_write_2d_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false) { - VK_LOG_DEBUG("ggml_vk_buffer_write_2d_async(" << width << ", " << height << ")"); - // Check if src is pinned memory - vk_buffer buf = nullptr; - size_t buf_offset = 0; - ggml_vk_host_get(dst->device, src, buf, buf_offset); - - if (buf != nullptr) { - // Memory is pinned, use as staging buffer - std::vector<vk::BufferCopy> slices(1); - if (width == spitch && width == dpitch) { - // Only do single write if stride is equal - slices[0].srcOffset = buf_offset; - slices[0].dstOffset = offset; - slices[0].size = width * height; + if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_F16) { + if (contig) { + return ctx->device->pipeline_contig_cpy_f32_f16; } else { - slices.resize(height); - for (size_t i = 0; i < height; i++) { - slices[i].srcOffset = buf_offset + i * spitch; - slices[i].dstOffset = offset + i * dpitch; - slices[i].size = width; - } + return ctx->device->pipeline_cpy_f32_f16; } - - ggml_vk_sync_buffers(nullptr, subctx); - subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices); - return true; - } - VK_LOG_DEBUG("STAGING"); - - if (!sync_staging) { - // copy was not handled caller needs to fall back - return false; } - - // Staging buffer required - const size_t staging_size = width * height; - ggml_vk_ensure_sync_staging_buffer(dst->device, staging_size); - - vk_buffer& staging_buffer = dst->device->sync_staging; - - std::vector<vk::BufferCopy> slices(1); - if (width == dpitch) { - slices[0].srcOffset = 0; - slices[0].dstOffset = offset; - slices[0].size = staging_size; - } else { - slices.resize(height); - for (size_t i = 0; i < height; i++) { - slices[i].srcOffset = i * width; - slices[i].dstOffset = offset + i * dpitch; - slices[i].size = width; + if (src->type == GGML_TYPE_F16 && to == GGML_TYPE_F16) { + if (contig) { + return ctx->device->pipeline_contig_cpy_f16_f16; + } else { + return ctx->device->pipeline_cpy_f16_f16; } } - - ggml_vk_sync_buffers(nullptr, subctx); - subctx->s->buffer->buf.copyBuffer((VkBuffer)staging_buffer->buffer, (VkBuffer)dst->buffer, slices); - - if (width == spitch) { - deferred_memcpy((uint8_t *)staging_buffer->ptr, src, staging_size, &subctx->in_memcpys); - } else { - for (size_t i = 0; i < height; i++) { - deferred_memcpy((uint8_t *)staging_buffer->ptr + i * width, (const uint8_t *) src + i * spitch, width, &subctx->in_memcpys); + if (src->type == GGML_TYPE_F16 && to == GGML_TYPE_F32) { + if (contig) { + return ctx->device->pipeline_contig_cpy_f16_f32; + } else { + return ctx->device->pipeline_cpy_f16_f32; } } - return true; -} - -static bool ggml_vk_buffer_write_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t size, bool sync_staging = false) { - VK_LOG_DEBUG("ggml_vk_buffer_write_async(" << size << ")"); - return ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, size, size, size, 1, sync_staging); -} - -static void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height) { - VK_LOG_DEBUG("ggml_vk_buffer_write_2d(" << width << ", " << height << ")"); - - // Invalidate the WP_VK_HOST_READ_FASTPATH read-side cache. This is a - // direct host->device write outside ggml_backend_vk_graph_compute()'s - // per-graph reset/republish dance (see the reset at the top of that - // function and the republish in its "submit || last_node" block), so if - // device->host_read_source still points at (a superset of) [offset, - // offset+width*height) in dst, a later ggml_vk_buffer_read_2d() would - // take the staged_host_read/host_cached fastpath and hand back bytes - // that predate THIS write. Concretely this bit wp-expert-worker's fused - // self-check: write_io_partial()'s ggml_backend_tensor_set() -> here -> - // left host_read_source pointing at the just-overwritten region, so the - // caller's next read_result() served the stale cached snapshot instead - // of the value this write just landed. Reset unconditionally rather than - // checking for overlap -- cheap (three field writes, no Vulkan calls) - // and always correct. - static const bool host_read_fastpath = [] { - const char * e = getenv("WP_VK_HOST_READ_FASTPATH"); - return e != nullptr && e[0] == '1'; - }(); - if (host_read_fastpath) { - dst->device->host_read_source.reset(); - dst->device->host_read_source_offset = 0; - dst->device->host_read_source_size = 0; - } - - // Buffer is already mapped - if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { - GGML_ASSERT(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent); - - if (width == spitch && width == dpitch) { - memcpy((uint8_t *)dst->ptr + offset, src, width * height); + if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_BF16) { + if (contig) { + return ctx->device->pipeline_contig_cpy_f32_bf16; } else { - for (size_t i = 0; i < height; i++) { - memcpy((uint8_t *)dst->ptr + offset + i * dpitch, (const uint8_t *) src + i * spitch, width); - } - } - } else { - std::lock_guard<std::recursive_mutex> guard(dst->device->mutex); - - vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(dst->device, subctx); - bool ret = ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, spitch, dpitch, width, height, true); - GGML_ASSERT(ret); - ggml_vk_ctx_end(subctx); - - for (auto& cpy : subctx->in_memcpys) { - memcpy(cpy.dst, cpy.src, cpy.n); - } - - for (auto& mset : subctx->memsets) { - memset(mset.dst, mset.val, mset.n); + return ctx->device->pipeline_cpy_f32_bf16; } - - ggml_vk_submit(subctx, dst->device->fence); - VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device); - dst->device->device.resetFences({ dst->device->fence }); - ggml_vk_queue_command_pools_cleanup(dst->device); } -} - -static void ggml_vk_buffer_write(vk_buffer& dst, size_t offset, const void * src, size_t size) { - VK_LOG_DEBUG("ggml_vk_buffer_write(" << size << ")"); - ggml_vk_buffer_write_2d(dst, offset, src, size, size, size, 1); -} - -static bool ggml_vk_buffer_read_2d_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false) { - VK_LOG_DEBUG("ggml_vk_buffer_read_2d_async(offset=" << offset << ", width=" << width << ", height=" << height << ")"); - GGML_ASSERT(width > 0); - GGML_ASSERT(height > 0); - GGML_ASSERT(src != nullptr); - - // TODO: staging_offset is not used - - // Check if dst is pinned memory - vk_buffer buf = nullptr; - size_t buf_offset = 0; - ggml_vk_host_get(src->device, dst, buf, buf_offset); - - std::vector<vk::BufferCopy> slices(1); - if (width == spitch && width == dpitch) { - // Only do single write if stride is equal - slices[0].srcOffset = offset; - slices[0].dstOffset = buf_offset; - slices[0].size = width * height; - } else { - slices.resize(height); - for (size_t i = 0; i < height; i++) { - slices[i].srcOffset = offset + i * spitch; - slices[i].dstOffset = buf_offset + i * dpitch; - slices[i].size = width; + if (src->type == GGML_TYPE_BF16 && to == GGML_TYPE_F32) { + if (contig) { + return ctx->device->pipeline_contig_cpy_bf16_f32; + } else { + return ctx->device->pipeline_cpy_bf16_f32; } } - - if (buf != nullptr) { - // Memory is pinned, use as staging buffer - ggml_vk_sync_buffers(nullptr, subctx); - subctx->s->buffer->buf.copyBuffer(src->buffer, buf->buffer, slices); - - return true; + if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_I32) { + if (contig) { + return ctx->device->pipeline_contig_cpy_f32_i32; + } else { + return ctx->device->pipeline_cpy_f32_i32; + } } - VK_LOG_DEBUG("STAGING"); - - if (!sync_staging) { - // copy was not handled caller needs to fall back - return false; + if (src->type == GGML_TYPE_I32 && to == GGML_TYPE_F32) { + if (contig) { + return ctx->device->pipeline_contig_cpy_i32_f32; + } else { + return ctx->device->pipeline_cpy_i32_f32; + } } - - // Fall back to staging buffer - const size_t staging_size = width * height; - ggml_vk_ensure_sync_staging_buffer(src->device, staging_size); - - vk_buffer& staging_buffer = src->device->sync_staging; - - std::vector<vk::BufferCopy> staging_slices(1); - if (width == spitch) { - staging_slices[0].srcOffset = offset; - staging_slices[0].dstOffset = 0; - staging_slices[0].size = staging_size; - } else { - staging_slices.resize(height); - for (size_t i = 0; i < height; i++) { - staging_slices[i].srcOffset = offset + i * spitch; - staging_slices[i].dstOffset = i * width; - staging_slices[i].size = width; + if (src->type == GGML_TYPE_F32) { + switch (to) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_TURBO4_0: + return ctx->device->pipeline_cpy_f32_quant[to]; + default: + break; } } - ggml_vk_sync_buffers(nullptr, subctx); - subctx->s->buffer->buf.copyBuffer(src->buffer, staging_buffer->buffer, staging_slices); - - if (width == dpitch) { - deferred_memcpy(dst, staging_buffer->ptr, staging_size, &subctx->out_memcpys); - } else { - for (size_t i = 0; i < height; i++) { - deferred_memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) staging_buffer->ptr + i * width, width, &subctx->out_memcpys); + if (to == GGML_TYPE_F32) { + switch (src->type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_IQ4_NL: + return ctx->device->pipeline_cpy_quant_f32[src->type]; + default: + break; } } - return true; -} - -static bool ggml_vk_buffer_read_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t size, bool sync_staging = false) { - return ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, size, size, size, 1, sync_staging); -} -static void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) { - VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")"); - - // Direct reads from host-visible device memory are opt-in because PCIe reads - // can be slower than a device copy for large results. - static const size_t direct_read_max_bytes = [] { - const char * e = getenv("GGML_VK_HOST_VISIBLE_DIRECT_READ_MAX_BYTES"); - return (e != nullptr && e[0] != '\0') ? (size_t) strtoull(e, nullptr, 10) : (size_t) 0; - }(); - const bool host_visible = - static_cast<bool>(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible); - const bool host_cached = - static_cast<bool>(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached); - const bool direct_host_read = host_visible && - (src->device->uma || - (direct_read_max_bytes != 0 && height != 0 && width <= direct_read_max_bytes / height)); - - // *** WP_VK_HOST_READ_FASTPATH -- the read half of the io-small BAR fix. *** - // - // ggml_vk_buffer_write_2d already treats a host-visible destination as a - // plain memcpy with no command buffer at all (see the eHostVisible branch - // there). The read side is not symmetric: even the direct_host_read branch - // below allocates a command buffer, records a single pipelineBarrier, - // vkQueueSubmits it and waits on a fence -- a full driver round trip for a - // 10 KB decode result. That asymmetry is the ~80 us/request ns_vk_readback - // on the RX 480. - // - // What the barrier submit actually buys is two things: (a) it makes prior - // device writes visible to the host, and (b) because the compute queue is - // in-order, its fence doubles as "wait for everything already submitted". - // (a) is redundant after ANY covering fence wait -- a fence signal operation - // carries a device->host memory domain operation, and this memory is - // asserted HOST_COHERENT below. (b) is redundant when the queue is already - // quiesced, which is exactly the wp-expert-worker's case: read_result runs - // after ggml_backend_synchronize has waited on ctx->fence. - // - // So when the queue epochs say nothing is outstanding, the whole round trip - // collapses to the same memcpy the write path uses. NO NUMERICS CHANGE -- - // the identical bytes are copied, only the wait that precedes the copy is - // dropped, and only when a stronger wait has already happened. - // - // Default OFF. The byte cap keeps large prefill readbacks on the staging - // copy (a big PCIe read from BAR is slower than a device copy into GTT). - static const bool host_read_fastpath = [] { - const char * e = getenv("WP_VK_HOST_READ_FASTPATH"); - return e != nullptr && e[0] == '1'; - }(); - static const size_t host_read_fastpath_max = [] { - const char * e = getenv("WP_VK_HOST_READ_FASTPATH_MAX_BYTES"); - return (e != nullptr && e[0] != '\0') ? (size_t) strtoull(e, nullptr, 10) : (size_t) 262144; - }(); - const bool fastpath_size_ok = height != 0 && width <= host_read_fastpath_max / height; - const bool staged_host_read = host_read_fastpath && fastpath_size_ok && !host_cached && - src == src->device->host_read_source && - offset >= src->device->host_read_source_offset && - offset - src->device->host_read_source_offset <= src->device->host_read_source_size && - width * height <= src->device->host_read_source_size - (offset - src->device->host_read_source_offset); - const vk_buffer& fastpath_src = staged_host_read ? src->device->host_read_staging : src; - const size_t fastpath_offset = staged_host_read ? offset - src->device->host_read_source_offset : offset; - if (host_read_fastpath && (host_cached || staged_host_read) && - (fastpath_src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) && - (fastpath_src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent) && - (fastpath_src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached) && - fastpath_size_ok && - ggml_vk_queue_quiesced(*src->device->compute_queue->handle)) { - if (width == spitch && width == dpitch) { - memcpy(dst, (const uint8_t *) fastpath_src->ptr + fastpath_offset, width * height); + if (src->type == to) { + // Copy two or four bytes at a time, depending on block size. + // For quantized types, we scale by block size/type size. But + // this path is also used for bf16->bf16 for example, where the + // type size must be exactly 2 or 4. + GGML_ASSERT(ggml_is_quantized(to) || ggml_type_size(src->type) == 2 || ggml_type_size(src->type) == 4); + if ((ggml_type_size(src->type) % 4) == 0) { + if (contig) { + return ctx->device->pipeline_contig_cpy_f32_f32; + } else { + return ctx->device->pipeline_cpy_f32_f32; + } } else { - for (size_t i = 0; i < height; i++) { - memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) fastpath_src->ptr + fastpath_offset + i * spitch, width); + if (contig) { + return ctx->device->pipeline_contig_cpy_f16_f16; + } else { + return ctx->device->pipeline_cpy_f16_f16; } } - return; } - if (direct_host_read) { - GGML_ASSERT(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent); - - std::lock_guard<std::recursive_mutex> guard(src->device->mutex); - vk_context subctx = ggml_vk_create_temporary_context(src->device->compute_queue->cmd_pool); - ggml_vk_ctx_begin(src->device, subctx); - subctx->s->buffer->buf.pipelineBarrier( - vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eHost, - {}, - { { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite, - vk::AccessFlagBits::eHostRead } }, - {}, {}); - ggml_vk_ctx_end(subctx); - ggml_vk_submit(subctx, src->device->fence); - const uint64_t epoch = ggml_vk_queue_epoch(*src->device->compute_queue->handle); - VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), - "vk_buffer_read_2d uma waitForFences", src->device); - src->device->device.resetFences({ src->device->fence }); - // This fence covers every earlier compute-queue submission (in-order queue). - ggml_vk_queue_mark_waited(*src->device->compute_queue->handle, epoch); - ggml_vk_queue_command_pools_cleanup(src->device); - - if (width == spitch && width == dpitch) { - memcpy(dst, (const uint8_t *) src->ptr + offset, width * height); - } else { - for (size_t i = 0; i < height; i++) { - memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) src->ptr + offset + i * spitch, width); - } - } - } else { - std::lock_guard<std::recursive_mutex> guard(src->device->mutex); + std::cerr << "Missing CPY op for types: " << ggml_type_name(src->type) << " " << ggml_type_name(to) << std::endl; + GGML_ABORT("fatal error"); +} - vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(src->device, subctx); - bool ret = ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, spitch, dpitch, width, height, true); - GGML_ASSERT(ret); - ggml_vk_ctx_end(subctx); +void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, const vk_subbuffer & in, const vk_subbuffer & out) { + VK_LOG_DEBUG("ggml_vk_cpy_to_contiguous((" << tensor << ", type=" << tensor->type << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), "; + std::cerr << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ")"); - ggml_vk_submit(subctx, src->device->fence); - VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device); - src->device->device.resetFences({ src->device->fence }); - ggml_vk_queue_command_pools_cleanup(src->device); + const uint32_t ne = ggml_nelements(tensor); + std::array<uint32_t, 3> elements; - for (auto& cpy : subctx->out_memcpys) { - memcpy(cpy.dst, cpy.src, cpy.n); - } + if (ne > 262144) { + elements = { 512, 512, CEIL_DIV(ne, 262144) }; + } else if (ne > 512) { + elements = { 512, CEIL_DIV(ne, 512), 1 }; + } else { + elements = { ne, 1, 1 }; } -} -static void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size) { - VK_LOG_DEBUG("ggml_vk_buffer_read(" << src->buffer << ", " << offset << ", " << size << ")"); - ggml_vk_buffer_read_2d(src, offset, dst, size, size, size, 1); + vk_op_unary_push_constants pc = vk_op_unary_push_constants_init(tensor, tensor, ne); + pc.nb10 = 1; + pc.nb11 = (uint32_t)tensor->ne[0]; + pc.nb12 = (uint32_t)(tensor->ne[0] * tensor->ne[1]); + pc.nb13 = (uint32_t)(tensor->ne[0] * tensor->ne[1] * tensor->ne[2]); + init_pushconst_fastdiv(pc); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { in, out }, pc, elements); + ggml_vk_sync_buffers(ctx, subctx); } -static void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) { - VK_LOG_DEBUG("ggml_vk_buffer_copy_async(" << size << ")"); - // Make sure both buffers are on same device - GGML_ASSERT(src->device == dst->device); - - VkBufferCopy bc{ src_offset, dst_offset, size }; +static void ggml_vk_cpy_to_strided( + ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, + const vk_subbuffer & in, const vk_subbuffer & out, + uint32_t nb10, uint32_t nb11, uint32_t nb12, uint32_t nb13) { + VK_LOG_DEBUG("ggml_vk_cpy_to_strided((" << tensor << ", type=" << tensor->type << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), "; + std::cerr << "dst_nb=(" << nb10 << ", " << nb11 << ", " << nb12 << ", " << nb13 << "), buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ")"); - vkCmdCopyBuffer(ctx->s->buffer->buf, (VkBuffer)src->buffer, (VkBuffer)dst->buffer, 1, &bc); -} + const uint32_t ne = ggml_nelements(tensor); + std::array<uint32_t, 3> elements; -static void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) { - if (src->device == dst->device) { - std::lock_guard<std::recursive_mutex> guard(src->device->mutex); - VK_LOG_DEBUG("ggml_vk_buffer_copy(SINGLE_DEVICE, " << size << ")"); - // Copy within the device - vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(src->device, subctx); - ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size); - ggml_vk_ctx_end(subctx); - ggml_vk_submit(subctx, src->device->fence); - VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device); - src->device->device.resetFences({ src->device->fence }); - ggml_vk_queue_command_pools_cleanup(src->device); + if (ne > 262144) { + elements = { 512, 512, CEIL_DIV(ne, 262144) }; + } else if (ne > 512) { + elements = { 512, CEIL_DIV(ne, 512), 1 }; } else { - VK_LOG_DEBUG("ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")"); - // Copy device to device - ggml_vk_ensure_sync_staging_buffer(src->device, size); - - // Copy to src staging buffer - ggml_vk_buffer_copy(src->device->sync_staging, 0, src, src_offset, size); - // Copy to dst buffer - ggml_vk_buffer_write(dst, dst_offset, src->device->sync_staging->ptr, size); + elements = { ne, 1, 1 }; } -} -static void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size) { - VK_LOG_DEBUG("ggml_vk_buffer_memset_async(" << offset << ", " << c << ", " << size << ")"); + vk_op_unary_push_constants pc = vk_op_unary_push_constants_init(tensor, tensor, ne); + pc.nb10 = nb10; + pc.nb11 = nb11; + pc.nb12 = nb12; + pc.nb13 = nb13; + init_pushconst_fastdiv(pc); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { in, out }, pc, elements); + ggml_vk_sync_buffers(ctx, subctx); +} - if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && - dst->device->uma) { - deferred_memset((uint8_t*)dst->ptr + offset, c, size, &ctx->memsets); - return; +vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type) { + switch(type) { + case GGML_TYPE_Q8_1: + return ctx->device->pipeline_quantize_q8_1_x4; + default: + std::cerr << "Missing quantize pipeline for type: " << ggml_type_name(type) << std::endl; + GGML_ABORT("fatal error"); } - - // Fall back to GPU fillBuffer for non-UMA or non-host-visible buffers - ctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c); } -static void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size) { - VK_LOG_DEBUG("ggml_vk_buffer_memset(" << offset << ", " << c << ", " << size << ")"); +void ggml_vk_quantize_q8_1(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_subbuffer & in, const vk_subbuffer & out, uint32_t ne) { + VK_LOG_DEBUG("ggml_vk_quantize_q8_1(" << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ", " << ne << ")"); - if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && - dst->device->uma) { - memset((uint8_t*)dst->ptr + offset, c, size); - return; - } + vk_pipeline pipeline = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); + + const uint32_t num_blocks = CEIL_DIV(ne, pipeline->wg_denoms[0]); + // clamp the number of elements to the max workgroup count. The shader will iterate over the total number of blocks. + const uint64_t max_elements = std::min<uint64_t>(uint64_t{ctx->device->properties.limits.maxComputeWorkGroupCount[0]} * pipeline->wg_denoms[0], std::numeric_limits<uint32_t>::max()); + const uint32_t elements = std::min(ne, static_cast<uint32_t>(max_elements)); - std::lock_guard<std::recursive_mutex> guard(dst->device->mutex); - vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(dst->device, subctx); - subctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c); - ggml_vk_ctx_end(subctx); + const vk_quantize_q8_1_push_constants pc = { + ne, + num_blocks, + }; - ggml_vk_submit(subctx, dst->device->fence); - VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device); - dst->device->device.resetFences({ dst->device->fence }); - ggml_vk_queue_command_pools_cleanup(dst->device); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { in, out }, pc, { elements, 1, 1 }); + ggml_vk_sync_buffers(ctx, subctx); } -struct ggml_vk_wp_event { - vk_device device; - vk_context context; - vk::Fence fence; -}; +static vk_pipeline ggml_vk_get_64b_indexing_pipeline(ggml_backend_vk_context * ctx, vk_pipeline &pipeline) { + GGML_UNUSED(ctx); +#if defined(VK_EXT_shader_64bit_indexing) + vk_pipeline *ptr = &pipeline; + while (*ptr) { + if ((*ptr)->is_64b_indexing) { + return *ptr; + } + ptr = &(*ptr)->next; + } +#endif + return pipeline; +} -// GGML_VK_WP_HASHCHK=1 debug scaffold. GGML_VK_WP_VERIFY proves the bytes are -// right when they are STAGED; it says nothing about whether they are still right -// when the dispatch that reads them is recorded. This registry closes that gap: -// stage_in records a hash per pool offset, and the paged mul_mat_id dispatch -// re-reads the bound pool at the offset it is about to hand the shader and -// compares. A mismatch means the slot was overwritten between stage-in and -// dispatch, i.e. a slot-lifetime bug. All matching means the data the shader -// reads is correct and the fault is elsewhere. -static std::mutex g_wp_hash_mutex; -static std::unordered_map<size_t, uint64_t> g_wp_slot_hash; +static constexpr uint32_t GGML_VK_MUL_MAT_FORCE_MM_REFERENCE_TOKENS = 512; +// pinned MUL_MAT default: the mat-vec shader in fixed groups of this many columns +static constexpr uint32_t GGML_VK_MUL_MAT_PIN_GROUP = 8; -static bool wp_hashchk_enabled() { - static const bool v = getenv("GGML_VK_WP_HASHCHK") != nullptr; - return v; -} +static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool disable_split_k, bool force_mm) { + VK_LOG_DEBUG("ggml_vk_mul_mat_q_f16((" << src0 << ", name=" << src0->name << ", type=" << ggml_type_name(src0->type) << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; + std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << ggml_type_name(src1->type) << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << ggml_type_name(dst->type) << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; + std::cerr << "))"); + GGML_ASSERT(ggml_vk_dim01_contiguous(src0) || src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); // NOLINT + GGML_ASSERT(ggml_vk_dim01_contiguous(src1) || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); // NOLINT -static uint64_t wp_hash_bytes(const void * p, size_t n) { - // FNV-1a; only needs to detect change, not resist anything. - uint64_t h = 0xcbf29ce484222325ull; - const uint8_t * b = (const uint8_t *) p; - for (size_t i = 0; i < n; ++i) { - h ^= b[i]; - h *= 0x100000001b3ull; - } - return h; -} + const uint64_t ne00 = src0->ne[0]; + const uint64_t ne01 = src0->ne[1]; + const uint64_t ne02 = src0->ne[2]; + const uint64_t ne03 = src0->ne[3]; -static constexpr size_t WP_HASH_SPAN = 4096; + const uint64_t ne10 = src1->ne[0]; + const uint64_t ne11 = src1->ne[1]; + const uint64_t ne12 = src1->ne[2]; + const uint64_t ne13 = src1->ne[3]; -bool ggml_backend_vk_wp_stage_in(ggml_backend_buffer_t buffer, - void * dst, const void * src, - size_t payload_size, size_t slot_size, - void ** event) { - if (buffer == nullptr || dst == nullptr || src == nullptr || event == nullptr || payload_size > slot_size) { - static std::atomic<bool> warned{false}; - if (!warned.exchange(true)) { - GGML_LOG_WARN("ggml_backend_vk_wp_stage_in: invalid arguments " - "(buffer=%p dst=%p src=%p event=%p payload=%zu slot=%zu)\n", - (void *) buffer, dst, src, (void *) event, payload_size, slot_size); - } - return false; - } + const uint64_t ne21 = dst->ne[1]; + const uint32_t stride_d = dst->nb[1] / ggml_type_size(dst->type); + const uint32_t stride_batch_d = stride_d*ne21; - auto * buffer_ctx = (ggml_backend_vk_buffer_context *) buffer->context; - if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { - static std::atomic<bool> warned{false}; - if (!warned.exchange(true)) { - GGML_LOG_WARN("ggml_backend_vk_wp_stage_in: missing Vulkan buffer context " - "(context=%p dev_buffer=%p)\n", - (void *) buffer_ctx, - buffer_ctx == nullptr ? nullptr : (void *) buffer_ctx->dev_buffer.get()); - } - return false; - } + const uint64_t r2 = ne12 / ne02; + const uint64_t r3 = ne13 / ne03; - vk_buffer dst_buffer = buffer_ctx->dev_buffer; - const uintptr_t base_addr = (uintptr_t) ggml_backend_buffer_get_base(buffer); - const uintptr_t dst_addr = (uintptr_t) dst; - const bool below_base = dst_addr < base_addr; - const size_t offset = below_base ? 0 : dst_addr - base_addr; - if (below_base || offset > dst_buffer->size || slot_size > dst_buffer->size - offset) { - static std::atomic<bool> warned{false}; - if (!warned.exchange(true)) { - GGML_LOG_WARN("ggml_backend_vk_wp_stage_in: destination outside pool buffer " - "(base=%p dst=%p offset=%zu slot=%zu buffer_size=%zu)\n", - (void *) base_addr, dst, offset, slot_size, dst_buffer->size); - } - return false; - } + // TODO: Clean up this logic to pick src1 type by capability + // Reformat and convert to fp16 if non-contiguous, or for coopmat2 for better perf + const bool x_non_contig = (ctx->device->coopmat2 && src0->type == GGML_TYPE_F32) || + !ggml_vk_dim01_contiguous(src0); + const bool y_non_contig = (ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) || + // coopmat1: force f32->f16 conversion so the f16 B-type quant pipeline is used. + (ctx->device->coopmat_support && !ctx->device->coopmat2 && + ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) || + (src0->type == GGML_TYPE_BF16 && src1->type != GGML_TYPE_BF16) || + !ggml_vk_dim01_contiguous(src1); - std::lock_guard<std::recursive_mutex> guard(dst_buffer->device->mutex); + // If src0 is BF16, try to use a BF16 x BF16 multiply + ggml_type f16_type = src0->type == GGML_TYPE_BF16 ? GGML_TYPE_BF16 : GGML_TYPE_F16; - // Can this transfer safely be left in flight? Only if `src` is host memory - // registered with THIS Vulkan device, in which case the copy reads straight - // from it. Otherwise ggml_vk_buffer_write_async routes through the single - // device-wide `device->sync_staging` buffer, and two overlapping transfers - // would each memcpy into that same region -- the second clobbering data the - // first's queued copy has not executed yet. So an unpinned source MUST be - // fenced before this function returns; the caller cannot know this, because - // the staging decision is made in here. - // - // The pager registers its O_DIRECT arena through - // ggml_backend_vk_wp_host_register. If import is unavailable, this lookup - // stays empty and the correctness-preserving fenced path remains active. - vk_buffer pinned_buf = nullptr; - size_t pinned_offset = 0; - ggml_vk_host_get(dst_buffer->device, src, pinned_buf, pinned_offset); - const bool src_is_pinned = pinned_buf != nullptr; + const bool y_f32_kernel = src1->type == GGML_TYPE_F32 && !y_non_contig; - // GGML_VK_WP_CQUEUE=1: stage on the COMPUTE queue instead of the transfer - // queue. On this device the two are separate queues (same family, 4 queues on - // RADV Polaris), and nothing orders a transfer-queue submission against the - // compute-queue dispatches that read the slot. Same queue == guaranteed order. - static const bool wp_cqueue = getenv("GGML_VK_WP_CQUEUE") != nullptr; - vk_context context = ggml_vk_create_temporary_context( - wp_cqueue ? dst_buffer->device->compute_queue->cmd_pool - : dst_buffer->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(dst_buffer->device, context); - if (!ggml_vk_buffer_write_async(context, dst_buffer, offset, src, payload_size, true)) { - static std::atomic<bool> warned{false}; - if (!warned.exchange(true)) { - GGML_LOG_WARN("ggml_backend_vk_wp_stage_in: ggml_vk_buffer_write_async returned false " - "(offset=%zu payload=%zu source_registered=%d)\n", - offset, payload_size, src_is_pinned ? 1 : 0); - } - return false; - } - if (slot_size > payload_size) { - ggml_vk_buffer_memset_async(context, dst_buffer, offset + payload_size, 0, slot_size - payload_size); - } - ggml_vk_ctx_end(context); + bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0; - for (auto & copy : context->in_memcpys) { - memcpy(copy.dst, copy.src, copy.n); - } - for (auto & memset : context->memsets) { - ::memset(memset.dst, memset.val, memset.n); + // Check for mmq first + const std::vector<vk_matmul_pipeline_pair>* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr; + + if (mmp_map == nullptr) { + // Fall back to f16 dequant mul mat + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]); + quantize_y = false; } - auto * wp_event = new ggml_vk_wp_event{ - dst_buffer->device, - context, - dst_buffer->device->device.createFence({}), - }; - ggml_vk_submit(context, wp_event->fence); + const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig; + const bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig); - // Shared-staging transfers cannot be left in flight -- see the note above. - if (!src_is_pinned) { - (void) ggml_backend_vk_wp_event_wait(wp_event); + if (qx_needs_dequant) { + // Fall back to dequant + f16 mulmat + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]); } - if (wp_hashchk_enabled()) { - const size_t n = std::min<size_t>(payload_size, WP_HASH_SPAN); - std::lock_guard<std::mutex> hg(g_wp_hash_mutex); - g_wp_slot_hash[offset] = wp_hash_bytes(src, n); - } + // Not implemented + GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT - // GGML_VK_WP_VERIFY=1: read the slot straight back and compare with the - // source. Answers "did the bytes actually land" without inference in the - // way. Debug only -- it stalls the queue on every page. - static const bool wp_verify = getenv("GGML_VK_WP_VERIFY") != nullptr; - if (wp_verify) { - (void) ggml_backend_vk_wp_event_wait(wp_event); - static int checked = 0, bad = 0; - const size_t n = std::min<size_t>(payload_size, 4096); - std::vector<uint8_t> back(n); - ggml_vk_buffer_read(dst_buffer, offset, back.data(), n); - if (memcmp(back.data(), src, n) != 0) { - if (bad < 4) { - size_t i = 0; - while (i < n && back[i] == ((const uint8_t *) src)[i]) { ++i; } - fprintf(stderr, "[vk-wp-verify] MISMATCH at slot offset %zu, first bad byte %zu: " - "got 0x%02x want 0x%02x (payload %zu)\n", - offset, i, back[i], ((const uint8_t *) src)[i], payload_size); - } - ++bad; - } - if (++checked % 500 == 0) { - fprintf(stderr, "[vk-wp-verify] %d pages checked, %d mismatched\n", checked, bad); + GGML_ASSERT(mmp_map != nullptr); + + // GGML_VK_PIN_REF_TOKENS overrides the pinned reference column count; 0 = follow the real ne11 + static const uint32_t pin_ref_tokens = [] { + const char * env = std::getenv("GGML_VK_PIN_REF_TOKENS"); + if (env == nullptr || env[0] == '\0') { + return GGML_VK_MUL_MAT_FORCE_MM_REFERENCE_TOKENS; } + const long value = std::strtol(env, nullptr, 10); + return value < 0 ? GGML_VK_MUL_MAT_FORCE_MM_REFERENCE_TOKENS : (uint32_t) value; + }(); + const uint32_t pipeline_n = force_mm && pin_ref_tokens > 0 ? pin_ref_tokens : (uint32_t) ne11; + + const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, pipeline_n, false)); + const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && pipeline_n > 8; + + vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, pipeline_n, aligned, false); + + if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { + pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); } - *event = wp_event; - return true; -} + // Reserve extra storage in the N dimension for the Y matrix, so we can avoid bounds-checking + uint32_t padded_n = qy_needs_dequant ? ROUNDUP_POW2(ne11, pipeline->wg_denoms[1]) : ne11; + const uint64_t x_ne = ggml_nelements(src0); + // 128 elements per Q8_1 x4 block + const uint64_t y_ne = padded_n * ne10 * ne12 * ne13; + const uint64_t d_ne = ggml_nelements(dst); -void * ggml_backend_vk_wp_host_alloc(ggml_backend_buffer_t pool_buffer, size_t size) { - if (pool_buffer == nullptr || size == 0) { - return nullptr; + const uint32_t split_k = force_mm ? 1 : ggml_vk_guess_split_k(ctx, ne01, ne11, ne10, disable_split_k, pipeline); + + const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type); + const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type); + const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne; + const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : (y_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne); + const uint64_t d_sz = sizeof(float) * d_ne; + + vk_pipeline to_fp16_vk_0 = nullptr; + vk_pipeline to_fp16_vk_1 = nullptr; + vk_pipeline to_q8_1 = nullptr; + + if (x_non_contig) { + to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0, nullptr, f16_type); + } else { + to_fp16_vk_0 = ggml_vk_get_to_fp16(ctx, src0->type); } - auto * buffer_ctx = (ggml_backend_vk_buffer_context *) pool_buffer->context; - if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { - return nullptr; + if (y_non_contig) { + to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, nullptr, f16_type); + } else { + to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); } - // Registered against the POOL's device, not vk_instance.devices[0], so - // ggml_vk_host_get finds it when staging into this pool. - return ggml_vk_host_malloc(buffer_ctx->dev_buffer->device, size); -} + GGML_ASSERT(!qx_needs_dequant || to_fp16_vk_0 != nullptr); // NOLINT + GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT -bool ggml_backend_vk_wp_host_register( - ggml_backend_buffer_t pool_buffer, void * ptr, size_t size) { - if (pool_buffer == nullptr || ptr == nullptr || size == 0) { - return false; - } - auto * buffer_ctx = (ggml_backend_vk_buffer_context *) pool_buffer->context; - if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { - return false; + if (quantize_y) { + to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); } - vk_device device = buffer_ctx->dev_buffer->device; - if (!device->external_memory_host) { - static std::atomic<bool> warned{false}; - if (!warned.exchange(true)) { - GGML_LOG_WARN("ggml_backend_vk_wp_host_register: device does not support " - "VK_EXT_external_memory_host\n"); + { + const uint64_t split_k_size = split_k > 1 ? d_sz * split_k : 0; + if ( + (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) || + (qy_needs_dequant && y_sz > ctx->device->properties.limits.maxStorageBufferRange) || + (split_k > 1 && split_k_size > ctx->device->properties.limits.maxStorageBufferRange)) { + GGML_ABORT("Requested preallocation size is too large"); } - return false; - } - const size_t import_alignment = device->min_imported_host_pointer_alignment; - if (import_alignment == 0 || - ((uintptr_t) ptr & (import_alignment - 1)) != 0 || - (size & (import_alignment - 1)) != 0) { - static std::atomic<bool> warned{false}; - if (!warned.exchange(true)) { - GGML_LOG_WARN("ggml_backend_vk_wp_host_register: host allocation does not meet " - "Vulkan import alignment (ptr=%p size=%zu alignment=%zu)\n", - ptr, size, import_alignment); + if (qx_needs_dequant && ctx->prealloc_size_x < x_sz) { + ctx->prealloc_size_x = x_sz; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if ((qy_needs_dequant || quantize_y) && ctx->prealloc_size_y < y_sz) { + ctx->prealloc_size_y = y_sz; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if (split_k > 1 && ctx->prealloc_size_split_k < split_k_size) { + ctx->prealloc_size_split_k = split_k_size; + ggml_vk_preallocate_buffers(ctx, subctx); } - return false; - } - vk_buffer imported = ggml_vk_buffer_from_host_ptr(device, ptr, size); - if (imported == nullptr) { - static std::atomic<bool> warned{false}; - if (!warned.exchange(true)) { - GGML_LOG_WARN("ggml_backend_vk_wp_host_register: Vulkan host pointer import failed " - "(ptr=%p size=%zu)\n", ptr, size); + // Request descriptor sets + if (qx_needs_dequant) { + ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_0, 1); + } + if (qy_needs_dequant) { + ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_1, 1); + } + if (quantize_y) { + ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1); + } + if (split_k > 1) { + ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, 1); } - return false; } - const uintptr_t begin = (uintptr_t) ptr; - const uintptr_t end = begin + size; - if (end < begin) { - return false; - } + ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; + ggml_backend_vk_buffer_context * src0_buf_ctx = (ggml_backend_vk_buffer_context *)src0->buffer->context; + ggml_backend_vk_buffer_context * src1_buf_ctx = (ggml_backend_vk_buffer_context *)src1->buffer->context; - std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex); - for (const auto & entry : device->pinned_memory) { - const uintptr_t registered_begin = (uintptr_t) std::get<0>(entry); - const uintptr_t registered_end = registered_begin + std::get<1>(entry); - if (begin < registered_end && registered_begin < end) { - return false; - } + vk_buffer d_D = dst_buf_ctx->dev_buffer; + const uint64_t d_buf_offset = vk_tensor_offset(dst) + dst->view_offs; + GGML_ASSERT(d_D != nullptr); + GGML_ASSERT(d_D->size >= d_buf_offset + d_sz); + vk_buffer d_Qx = nullptr; + size_t qx_buf_offset = 0; + vk_buffer d_Qy = nullptr; + size_t qy_buf_offset = 0; + bool src0_uma = false; + bool src1_uma = false; + + if (ctx->device->uma) { + ggml_vk_host_get(ctx->device, src0->data, d_Qx, qx_buf_offset); + ggml_vk_host_get(ctx->device, src1->data, d_Qy, qy_buf_offset); + src0_uma = d_Qx != nullptr; + src1_uma = d_Qy != nullptr; + } + if (!src0_uma) { + d_Qx = src0_buf_ctx->dev_buffer; + qx_buf_offset = vk_tensor_offset(src0) + src0->view_offs; + GGML_ASSERT(d_Qx != nullptr); + } + if (!src1_uma) { + d_Qy = src1_buf_ctx->dev_buffer; + qy_buf_offset = vk_tensor_offset(src1) + src1->view_offs; + GGML_ASSERT(d_Qy != nullptr); } - device->pinned_memory.push_back(std::make_tuple(ptr, size, std::move(imported))); - return true; -} -void ggml_backend_vk_wp_host_unregister( - ggml_backend_buffer_t pool_buffer, void * ptr) { - if (pool_buffer == nullptr || ptr == nullptr) { - return; + vk_subbuffer d_Qx_copy = ggml_vk_mark_dynamic_subbuffer( + ctx, ggml_vk_subbuffer(ctx, d_Qx, qx_buf_offset), src0); + vk_subbuffer d_Qy_copy = ggml_vk_mark_dynamic_subbuffer( + ctx, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), src1); + vk_subbuffer d_Qx_dequant = ggml_vk_mark_dynamic_subbuffer( + ctx, { d_Qx, qx_buf_offset, qx_sz }, src0); + vk_subbuffer d_Qy_dequant = ggml_vk_mark_dynamic_subbuffer( + ctx, { d_Qy, qy_buf_offset, qy_sz }, src1); + vk_subbuffer d_D_buf = ggml_vk_mark_dynamic_subbuffer( + ctx, ggml_vk_subbuffer(ctx, d_D, d_buf_offset), dst); + + vk_buffer d_X; + uint64_t x_buf_offset = 0; + vk_buffer d_Y; + uint64_t y_buf_offset = 0; + if (qx_needs_dequant) { + d_X = ctx->prealloc_x; + GGML_ASSERT(d_X->size >= x_sz); + } else { + d_X = d_Qx; + x_buf_offset = qx_buf_offset; + GGML_ASSERT(qx_sz == x_sz); } - auto * buffer_ctx = (ggml_backend_vk_buffer_context *) pool_buffer->context; - if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { - return; + if (qy_needs_dequant) { + d_Y = ctx->prealloc_y; + GGML_ASSERT(d_Y->size >= y_sz); + } else if (quantize_y) { + d_Y = ctx->prealloc_y; + GGML_ASSERT(d_Y->size >= CEIL_DIV(y_sz, 144) * 144); + } else { + d_Y = d_Qy; + y_buf_offset = qy_buf_offset; + GGML_ASSERT(qy_sz == y_sz); } - ggml_vk_host_free(buffer_ctx->dev_buffer->device, ptr); -} -bool ggml_backend_vk_wp_read(ggml_backend_buffer_t pool_buffer, - const void * src, void * dst, size_t n) { - if (pool_buffer == nullptr || src == nullptr || dst == nullptr || n == 0) { - return false; + vk_subbuffer d_X_buf = { d_X, x_buf_offset, x_sz }; + vk_subbuffer d_Y_buf = { d_Y, y_buf_offset, y_sz }; + if (!qx_needs_dequant) { + d_X_buf = ggml_vk_mark_dynamic_subbuffer(ctx, d_X_buf, src0); } - auto * buffer_ctx = (ggml_backend_vk_buffer_context *) pool_buffer->context; - if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { - return false; + if (!qy_needs_dequant && !quantize_y) { + d_Y_buf = ggml_vk_mark_dynamic_subbuffer(ctx, d_Y_buf, src1); } - vk_buffer src_buffer = buffer_ctx->dev_buffer; - const size_t offset = (uintptr_t) src - (uintptr_t) vk_ptr_base; - if (offset > src_buffer->size || n > src_buffer->size - offset) { - return false; + + if (x_non_contig || qx_needs_dequant) { + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } } - ggml_vk_buffer_read(src_buffer, offset, dst, n); - return true; -} -void ggml_backend_vk_wp_host_free(ggml_backend_buffer_t pool_buffer, void * ptr) { - if (pool_buffer == nullptr || ptr == nullptr) { - return; + if (x_non_contig) { + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, + d_Qx_copy, ggml_vk_subbuffer(ctx, d_X, 0)); + } else if (qx_needs_dequant) { + const std::vector<uint32_t> pc = { (uint32_t)ne01, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)(ggml_nelements(src0)) }; + ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, { d_Qx_dequant, { d_X, 0, x_sz } }, pc, { (uint32_t)(x_ne), 1, 1}); + ggml_vk_sync_buffers(ctx, subctx); } - auto * buffer_ctx = (ggml_backend_vk_buffer_context *) pool_buffer->context; - if (buffer_ctx == nullptr || buffer_ctx->dev_buffer == nullptr) { - return; + if (y_non_contig) { + if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || + ctx->prealloc_y_last_tensor_used != src1 || + ctx->prealloc_y_last_k_padded) { + if (ctx->prealloc_y_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, + d_Qy_copy, ggml_vk_subbuffer(ctx, d_Y, 0)); + ctx->prealloc_y_last_pipeline_used = to_fp16_vk_1.get(); + ctx->prealloc_y_last_tensor_used = src1; + ctx->prealloc_y_last_k_padded = false; + } + } + if (quantize_y) { + if (ctx->prealloc_y_last_pipeline_used != to_q8_1.get() || + ctx->prealloc_y_last_tensor_used != src1 || + ctx->prealloc_y_last_k_padded) { + if (ctx->prealloc_y_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + ggml_vk_quantize_q8_1(ctx, subctx, d_Qy_copy, ggml_vk_subbuffer(ctx, d_Y, 0), y_ne); + ctx->prealloc_y_last_pipeline_used = to_q8_1.get(); + ctx->prealloc_y_last_tensor_used = src1; + ctx->prealloc_y_last_k_padded = false; + } } - ggml_vk_host_free(buffer_ctx->dev_buffer->device, ptr); -} -// Weight-paging consumption bridge. Mirrors the CUDA backend's -// ggml_cuda_set_routed_expert_ptrs: the pager publishes where the active -// experts actually live, and the very next mul_mat_id dispatch consumes it. -// Thread-local for the same reason CUDA's is — eval runs on the compute thread -// and a publication must not leak across threads. -struct vk_wp_expert_offsets { - ggml_backend_buffer_t pool_buffer = nullptr; - std::vector<uint32_t> offsets; - bool valid = false; -}; -static thread_local vk_wp_expert_offsets tls_wp_expert_offsets; + uint32_t stride_batch_x = ne00*ne01; + uint32_t stride_batch_y = ne10*ne11; -void ggml_backend_vk_wp_set_expert_offsets(ggml_backend_buffer_t pool_buffer, - const uint32_t * block_offsets, - int n_experts) { - if (pool_buffer == nullptr || block_offsets == nullptr || n_experts <= 0) { - tls_wp_expert_offsets.valid = false; - return; + if (!ggml_vk_dim01_contiguous(src0) && !qx_needs_dequant) { + stride_batch_x = src0->nb[0] / ggml_type_size(src0->type); } - tls_wp_expert_offsets.pool_buffer = pool_buffer; - tls_wp_expert_offsets.offsets.assign(block_offsets, block_offsets + n_experts); - tls_wp_expert_offsets.valid = true; -} -bool ggml_backend_vk_wp_fused_expert( - ggml_backend_t backend, - const struct ggml_backend_vk_wp_fused_expert_params * params, - struct ggml_backend_vk_wp_fused_expert_stats * stats) { - if (stats != nullptr) { - *stats = {}; - } - if (backend == nullptr || !ggml_backend_is_vk(backend) || - params == nullptr || stats == nullptr || - params->io_buffer == nullptr || params->weights_buffers == nullptr || - params->params_buffer == nullptr || params->gate_byte_offsets == nullptr || - params->up_byte_offsets == nullptr || params->down_byte_offsets == nullptr || - params->route_weights == nullptr || params->n_tokens == 0 || - params->n_tokens > 8 || params->n_experts == 0 || params->n_ff == 0 || - params->n_ff > 256 || params->n_embd == 0 || - (params->down_type != GGML_TYPE_Q5_1 && params->down_type != GGML_TYPE_Q8_0)) { - return false; + if (!ggml_vk_dim01_contiguous(src1) && !qy_needs_dequant && !quantize_y) { + stride_batch_y = src1->nb[0] / ggml_type_size(src1->type); } - auto * ctx = (ggml_backend_vk_context *) backend->context; - auto * io_ctx = (ggml_backend_vk_buffer_context *) params->io_buffer->context; - auto * params_ctx = (ggml_backend_vk_buffer_context *) params->params_buffer->context; - if (ctx == nullptr || io_ctx == nullptr || - params_ctx == nullptr || io_ctx->dev_buffer == nullptr || - params_ctx->dev_buffer == nullptr || - io_ctx->dev_buffer->device != ctx->device || - params_ctx->dev_buffer->device != ctx->device || - params->n_experts > 4096) { - return false; - } + // compute + ggml_vk_matmul( + ctx, subctx, pipeline, + std::move(d_X_buf), std::move(d_Y_buf), std::move(d_D_buf), { ctx->prealloc_split_k, 0, d_sz * split_k }, + ne01, ne11, ne10, + ne10, ne10, stride_d, stride_batch_x, stride_batch_y, stride_batch_d, + split_k, ne12*ne13, ne02, ne12, r2, r3, padded_n + ); // NOLINT - const size_t n_values = (size_t) params->n_tokens * params->n_embd; - if (n_values > std::numeric_limits<size_t>::max() / sizeof(float)) { - return false; + if (x_non_contig || qx_needs_dequant) { + ctx->prealloc_x_need_sync = true; } - const size_t io_bytes = n_values * sizeof(float); - const size_t io_size = io_ctx->dev_buffer->size; - if (params->input_offset > io_size || io_bytes > io_size - params->input_offset || - params->output_offset > io_size || io_bytes > io_size - params->output_offset) { - return false; + if (y_non_contig || quantize_y) { + ctx->prealloc_y_need_sync = true; } +} - const size_t gate_blocks = ((size_t) params->n_embd + 255) / 256; - const size_t down_blocks = ((size_t) params->n_ff + 31) / 32; - if (gate_blocks == 0 || down_blocks == 0 || - (size_t) params->n_ff > std::numeric_limits<size_t>::max() / gate_blocks || - (size_t) params->n_embd > std::numeric_limits<size_t>::max() / down_blocks) { +static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_t n, uint32_t k, ggml_type src0_type) { + if (device->mmvq_mode == 1) { + return true; + } else if (device->mmvq_mode == -1) { return false; } - const size_t gate_block_bytes = 144; - const size_t down_block_bytes = params->down_type == GGML_TYPE_Q8_0 ? 34 : 24; - // Bytes spanned by one expert's gate/up role (they share a shape) and by - // one expert's down role. Each role is bound as its OWN sub-buffer at - // its own byte offset in the dispatch loop below, so these are also the - // descriptor ranges -- NOT the whole weights buffer. - if ((uint64_t) params->n_ff > std::numeric_limits<uint64_t>::max() / gate_blocks || - (uint64_t) params->n_ff * gate_blocks > - std::numeric_limits<uint64_t>::max() / gate_block_bytes) { + + // q6_k only has 2-byte alignment which makes it somewhat problematic, + // using MMVQ is only a win on Intel. + bool mmvq_q6 = device->vendor_id == VK_VENDOR_ID_INTEL; + if (src0_type == GGML_TYPE_Q6_K && !mmvq_q6) { return false; } - const uint64_t gate_role_bytes = (uint64_t) params->n_ff * gate_blocks * gate_block_bytes; - if ((uint64_t) params->n_embd > std::numeric_limits<uint64_t>::max() / down_blocks || - (uint64_t) params->n_embd * down_blocks > - std::numeric_limits<uint64_t>::max() / down_block_bytes) { - return false; + + // MMVQ is generally good for batches + if (n > 1) { + return true; } - const uint64_t down_role_bytes = (uint64_t) params->n_embd * down_blocks * down_block_bytes; - // An expert slot's position inside the pool buffer is only guaranteed - // aligned to this value (see layout_sliced_pages()/arena_slot_stride() - // in wp-expert-worker.cpp, both of which key off - // ggml_backend_buft_get_alignment() == minStorageBufferOffsetAlignment - // for the Vulkan buffer type), not to the coarser quant block size. - const VkDeviceSize align = ctx->device->properties.limits.minStorageBufferOffsetAlignment; + // Quantization overhead is not worth it for small k + switch (device->vendor_id) { + case VK_VENDOR_ID_NVIDIA: + if (src0_type == GGML_TYPE_Q2_0 || src0_type == GGML_TYPE_Q2_K || src0_type == GGML_TYPE_Q3_K || src0_type == GGML_TYPE_IQ1_S || src0_type == GGML_TYPE_IQ1_M) { + return true; + } - std::vector<vk_buffer> weight_buffers(params->n_experts); - std::vector<size_t> weight_sizes(params->n_experts); - for (uint32_t e = 0; e < params->n_experts; ++e) { - if (params->weights_buffers[e] == nullptr) { + if (k <= 4096) { return false; } - auto * weights_ctx = (ggml_backend_vk_buffer_context *) - params->weights_buffers[e]->context; - if (weights_ctx == nullptr || weights_ctx->dev_buffer == nullptr || - weights_ctx->dev_buffer->device != ctx->device) { + + switch (src0_type) { + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q8_0: + return device->architecture == vk_device_architecture::NVIDIA_PRE_TURING; + default: + return true; + } + case VK_VENDOR_ID_AMD: + if (k < 2048) { return false; } - weight_buffers[e] = weights_ctx->dev_buffer; - weight_sizes[e] = weights_ctx->dev_buffer->size; - const uint64_t gate_off = params->gate_byte_offsets[e]; - const uint64_t up_off = params->up_byte_offsets[e]; - const uint64_t down_off = params->down_byte_offsets[e]; - // The real alignment gate: binding a role's sub-buffer at a - // misaligned offset is a Vulkan validity violation, not merely - // slow. gate/up/down are validated separately since nothing - // guarantees they share a phase. - if (align == 0 || gate_off % align != 0 || up_off % align != 0 || - down_off % align != 0) { + + switch (src0_type) { + case GGML_TYPE_Q8_0: + return device->architecture == vk_device_architecture::AMD_GCN; + default: + return true; + } + case VK_VENDOR_ID_INTEL: + if (device->architecture == vk_device_architecture::INTEL_XE2) { + if (src0_type == GGML_TYPE_Q2_0 || src0_type == GGML_TYPE_Q2_K || src0_type == GGML_TYPE_Q3_K || src0_type == GGML_TYPE_Q6_K) { + return true; + } + } + + if (device->driver_id == vk::DriverId::eIntelProprietaryWindows) { + // Intel Windows proprietary driver MMVQ performance for !Q2/Q3/Q6 is worse than fp16, + // see https://github.com/ggml-org/llama.cpp/issues/17628 and + // https://github.com/ggml-org/llama.cpp/pull/23056 return false; } - if (gate_off > weight_sizes[e] || gate_role_bytes > weight_sizes[e] - gate_off || - up_off > weight_sizes[e] || gate_role_bytes > weight_sizes[e] - up_off || - down_off > weight_sizes[e] || down_role_bytes > weight_sizes[e] - down_off) { + + if (k < 2048) { + return false; + } + + switch (src0_type) { + // From tests on A770 Linux, may need more tuning + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_IQ4_XS: return false; + default: + return true; } + default: + return true; } - const size_t header_words = 9; - const size_t route_words = (size_t) params->n_experts > - std::numeric_limits<size_t>::max() / params->n_tokens - ? 0 : (size_t) params->n_experts * params->n_tokens; - if (route_words == 0 || route_words > std::numeric_limits<size_t>::max() - header_words) { - return false; + GGML_UNUSED(m); +} + +static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, bool swap_inputs = false, bool pin = false) { + ggml_tensor * dst = cgraph->nodes[node_idx]; + const ggml_tensor * src0 = dst->src[swap_inputs ? 1 : 0]; + const ggml_tensor * src1 = dst->src[swap_inputs ? 0 : 1]; + + VK_LOG_DEBUG("ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; + std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; + std::cerr << ")),)"); + GGML_ASSERT(ggml_vk_dim01_contiguous(src0) || src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); // NOLINT + GGML_ASSERT(ggml_vk_dim01_contiguous(src1) || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); // NOLINT + + const uint64_t ne00 = src0->ne[0]; + const uint64_t ne01 = src0->ne[1]; + const uint64_t ne02 = src0->ne[2]; + const uint64_t ne03 = src0->ne[3]; + + const uint64_t ne10 = src1->ne[0]; + const uint64_t ne11 = src1->ne[1]; + const uint64_t ne12 = src1->ne[2]; + const uint64_t ne13 = src1->ne[3]; + + const uint64_t ne20 = dst->ne[swap_inputs ? 1 : 0]; + const uint64_t ne21 = dst->ne[swap_inputs ? 0 : 1]; + // const uint64_t ne22 = dst->ne[2]; + // const uint64_t ne23 = dst->ne[3]; + + const uint64_t r2 = ne12 / ne02; + const uint64_t r3 = ne13 / ne03; + + // batch_n indicates that we need to compute a few vector results, and this assumes + // ne12 and ne13 are 1. It overloads the batch_strides to hold the row strides. + GGML_ASSERT(ne11 == 1 || ne12 * ne13 == 1); + bool batch_n = ne11 > 1; + + const bool x_non_contig = !ggml_vk_dim01_contiguous(src0); + const bool y_non_contig = !ggml_vk_dim01_contiguous(src1); + + const bool f16_f32_kernel = src1->type == GGML_TYPE_F32; + // pinned (GGML_HINT_MUL_MAT_PIN) groups evaluate the mmvq decision at the fixed group width, so the + // y-quantization choice cannot change with the real column count + const uint64_t ne11_decide = pin ? GGML_VK_MUL_MAT_PIN_GROUP : ne11; + bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11_decide * ne10) % 4 == 0 && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11_decide, ne10, src0->type); + + vk_pipeline to_fp16_vk_0 = nullptr; + vk_pipeline to_fp16_vk_1 = nullptr; + if (x_non_contig) { + to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0, nullptr, src0->type); } - const size_t total_words = header_words + route_words; - const size_t upload_bytes = total_words * sizeof(uint32_t); - if (upload_bytes / sizeof(uint32_t) != total_words || - params->params_offset > params_ctx->dev_buffer->size || - upload_bytes > params_ctx->dev_buffer->size - params->params_offset) { - return false; + if (y_non_contig) { + to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, nullptr, src1->type); + } else { + to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); } - const VkDeviceSize max_storage_buffer_range = - ctx->device->properties.limits.maxStorageBufferRange; - if ((VkDeviceSize) io_bytes > max_storage_buffer_range || - (VkDeviceSize) upload_bytes > max_storage_buffer_range || - (VkDeviceSize) gate_role_bytes > max_storage_buffer_range || - (VkDeviceSize) down_role_bytes > max_storage_buffer_range) { - return false; + // Check for mmq first + vk_pipeline dmmv = quantize_y ? ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, GGML_TYPE_Q8_1, ne11, ne20, ne00) : nullptr; + vk_pipeline to_q8_1 = nullptr; + + if (dmmv == nullptr) { + // Fall back to f16 dequant mul mat + dmmv = ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, src1->type, ne11, ne20, ne00); + quantize_y = false; } - const bool down_q8 = params->down_type == GGML_TYPE_Q8_0; - vk_pipeline pipeline = down_q8 - ? ctx->device->pipeline_wp_fused_expert_q8_0 - : ctx->device->pipeline_wp_fused_expert_q5_1; - // Batch fast path is now two pipelines (see the struct member comment on - // pipeline_wp_fused_expert_batch_a): phase A has no down-type variant, - // phase B does. - vk_pipeline batch_pipeline_a = ctx->device->pipeline_wp_fused_expert_batch_a; - vk_pipeline batch_pipeline_b = down_q8 - ? ctx->device->pipeline_wp_fused_expert_batch_b_q8_0 - : ctx->device->pipeline_wp_fused_expert_batch_b_q5_1; - if (ctx->device->subgroup_size != 64 || !ctx->device->subgroup_arithmetic || - ctx->device->properties.limits.maxComputeWorkGroupInvocations < 512 || - pipeline == nullptr) { - return false; + if (quantize_y) { + to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); } - // *** WP_VK_FUSED_EXPERT_BATCH: single-request, whole-GPU fast path. *** - // - // The per-expert loop below launches n_tokens (1-4 at decode) workgroups - // PER EXPERT -- one CU busy, the other 35 idle on this 36-CU part, and - // experts run one after another because each dispatch folds into the - // SAME output range the previous one wrote (a real RAW hazard, hence the - // barrier at the bottom of that loop). wp_fused_expert_batch_a.comp / - // wp_fused_expert_batch_b.comp fix both problems at once: phase A - // computes silu(gate)*up for every selected expert/token/hidden-unit in - // ONE dispatch (grid = n_experts * n_tokens * ceil(n_ff/8), i.e. ~200+ - // workgroups at 9 experts instead of 1), phase B then does the - // down-projection and folds across experts INSIDE one shader invocation - // per output row (grid = n_tokens * ceil(n_embd/8)) instead of across - // dispatches, so there is only one inter-phase barrier total (the - // ggml_vk_sync_buffers call between the two dispatches below), not one - // per expert. Unlike the earlier single-shader design, phase B contains - // no barrier() of its own and no shared-memory staging -- see - // wp_fused_expert_batch_b.comp's top-of-file comment for why that - // mattered (it hung this device via ErrorDeviceLost). - // - // The catch: these shaders bind gate/up/down as ONE whole-arena storage - // buffer per role and compute each expert's byte base in-shader (see - // FUSED_BATCH_MAX_EXPERTS below), so every selected expert must live in - // the SAME VkBuffer. That is NOT guaranteed in general -- Vulkan's - // allocate_slot_arenas() (wp-expert-worker.cpp) caps each arena at - // ggml_backend_buft_get_max_size() and splits any bigger pool across - // several arenas, and a request's selected experts can land in - // different ones -- so this is checked per call, not assumed, and falls - // through to the unchanged per-expert loop whenever it doesn't hold. - constexpr uint32_t FUSED_BATCH_MAX_EXPERTS = 16; - // Must match wp_fused_expert_batch_a.comp/wp_fused_expert_batch_b.comp's - // compile-time MAX_FF / MAX_TOKENS / MAX_EMBD caps -- every shader loop - // is bounded by these IN ADDITION TO its runtime count, and every value - // read from the params buffer that feeds a loop bound is clamped to the - // matching MAX_* again in-shader. FUSED_BATCH_MAX_FF/MAX_EXPERTS also - // size the scratch buffer below (n_experts * n_tokens * n_ff floats), - // which phase B now reads directly (no more shared-memory hidden cache, - // so there is no maxComputeSharedMemorySize gate to check here any - // more). - constexpr size_t FUSED_BATCH_MAX_FF = 256; - constexpr uint32_t FUSED_BATCH_MAX_TOKENS = 8; - constexpr uint32_t FUSED_BATCH_MAX_EMBD = 4096; - // Kill switch for the batch geometry only, independent of the other - // WP_VK_* env reads above: default on (1), set to 0 to force every - // request through the unchanged per-expert dispatch loop below without - // a rebuild, so the two geometries can be A/B'd against each other. - static const bool batch_geometry_enabled = [] { - const char * e = getenv("WP_VK_FUSED_EXPERT_BATCH"); - return e == nullptr || e[0] != '0'; - }(); - bool batch_fast_path = batch_geometry_enabled && - batch_pipeline_a != nullptr && batch_pipeline_b != nullptr && - params->n_experts <= FUSED_BATCH_MAX_EXPERTS && - params->n_tokens <= FUSED_BATCH_MAX_TOKENS && - params->n_ff <= FUSED_BATCH_MAX_FF && - params->n_embd <= FUSED_BATCH_MAX_EMBD && - (VkDeviceSize) weight_sizes[0] <= max_storage_buffer_range; - for (uint32_t e = 1; batch_fast_path && e < params->n_experts; ++e) { - // weight_buffers[e] is a shared_ptr<vk_buffer_struct>; equality here - // compares the pointee, i.e. "is this the same VkBuffer" -- exactly - // the identity the whole-buffer binding needs. - if (weight_buffers[e] != weight_buffers[0]) { - batch_fast_path = false; - } + if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { + dmmv = ggml_vk_get_64b_indexing_pipeline(ctx, dmmv); } - std::lock_guard<std::recursive_mutex> guard(ctx->device->mutex); - const auto started = std::chrono::steady_clock::now(); - const bool submit_async = params->submit_async != 0 && !ctx->device->serialize_submissions; - vk_context compute_ctx = ggml_vk_get_compute_ctx(ctx); - uint64_t n_pipeline_dispatches = 0; - uint64_t n_workgroups = 0; - uint64_t bytes_uploaded_actual = 0; + const bool qx_needs_dequant = x_non_contig; + const bool qy_needs_dequant = !quantize_y && ((src1->type != GGML_TYPE_F16 && !f16_f32_kernel) || y_non_contig); - if (batch_fast_path) { - // Header layout private to the batch shader (10 words): unlike the - // per-expert path's header, gate/up/down bases travel in the params - // buffer too (as BYTE offsets into the single whole-arena binding), - // since there is no per-expert sub-buffer rebind to carry them - // implicitly. n_embd/n_ff fully determine gate_blocks/down_blocks, - // so those are recomputed in-shader instead of uploaded. - const size_t batch_header_words = 10; - const size_t batch_route_words = route_words; // already validated != 0, no overflow - if (params->n_experts > std::numeric_limits<size_t>::max() - - (batch_header_words + batch_route_words)) { - return false; + // Not implemented + GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT + + GGML_ASSERT(!qx_needs_dequant || to_fp16_vk_0 != nullptr); // NOLINT + GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT + GGML_ASSERT(dmmv != nullptr); + + const uint64_t x_ne = ggml_nelements(src0); + const uint64_t y_ne = ggml_nelements(src1); + + const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device->properties.limits.minStorageBufferOffsetAlignment); + const uint64_t x_sz = x_non_contig ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : qx_sz; + const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : + (f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne); + + { + if ( + (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) || + (qy_needs_dequant && y_sz > ctx->device->properties.limits.maxStorageBufferRange)) { + GGML_ABORT("Requested preallocation size is too large"); } - const size_t gate_off_base = batch_header_words + batch_route_words; - const size_t up_off_base = gate_off_base + params->n_experts; - const size_t down_off_base = up_off_base + params->n_experts; - const size_t batch_total_words = down_off_base + params->n_experts; - const size_t batch_upload_bytes = batch_total_words * sizeof(uint32_t); - if (batch_upload_bytes / sizeof(uint32_t) != batch_total_words || - params->params_offset > params_ctx->dev_buffer->size || - batch_upload_bytes > params_ctx->dev_buffer->size - params->params_offset || - (VkDeviceSize) batch_upload_bytes > max_storage_buffer_range) { - return false; + if (qx_needs_dequant && ctx->prealloc_size_x < x_sz) { + ctx->prealloc_size_x = x_sz; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if ((qy_needs_dequant || quantize_y) && ctx->prealloc_size_y < y_sz) { + ctx->prealloc_size_y = y_sz; + ggml_vk_preallocate_buffers(ctx, subctx); } - // Per-expert BYTE offsets fit in uint32 without a lo/hi split: the - // whole-arena binding's range is <= max_storage_buffer_range, and - // VkPhysicalDeviceLimits::maxStorageBufferRange is itself a - // uint32_t field in the Vulkan spec, so any offset within that - // binding is representable in 32 bits. - std::vector<uint32_t> batch_upload(batch_total_words, 0); - batch_upload[0] = params->n_tokens; - batch_upload[1] = params->n_embd; - batch_upload[2] = params->n_ff; - batch_upload[3] = params->n_experts; - batch_upload[4] = params->add_previous != 0; - std::memcpy(&batch_upload[5], ¶ms->swiglu_limit, sizeof(float)); - batch_upload[6] = (uint32_t) batch_header_words; // route_weights_base - batch_upload[7] = (uint32_t) gate_off_base; - batch_upload[8] = (uint32_t) up_off_base; - batch_upload[9] = (uint32_t) down_off_base; - for (uint32_t e = 0; e < params->n_experts; ++e) { - for (uint32_t token = 0; token < params->n_tokens; ++token) { - std::memcpy(&batch_upload[batch_header_words + (size_t) e * params->n_tokens + token], - ¶ms->route_weights[(size_t) e * params->n_tokens + token], - sizeof(float)); - } - batch_upload[gate_off_base + e] = (uint32_t) params->gate_byte_offsets[e]; - batch_upload[up_off_base + e] = (uint32_t) params->up_byte_offsets[e]; - batch_upload[down_off_base + e] = (uint32_t) params->down_byte_offsets[e]; + // Request descriptor sets + if (qx_needs_dequant) { + ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_0, 1); } + if (qy_needs_dequant) { + ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_1, 1); + } + if (quantize_y) { + ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1); + } + } - const size_t scratch_floats = - (size_t) params->n_experts * params->n_tokens * params->n_ff; - const size_t scratch_bytes = scratch_floats * sizeof(float); - if (scratch_floats / params->n_tokens / params->n_experts != params->n_ff || - !ggml_vk_ensure_wp_fused_batch_scratch_buffer(ctx->device, scratch_bytes)) { - return false; // scratch OOM: fail the call rather than silently mis-fusing + vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, cgraph->nodes[node_idx + ctx->num_additional_fused_ops]); + vk_subbuffer d_Qx = ggml_vk_tensor_subbuffer(ctx, src0); + vk_subbuffer d_Qy = ggml_vk_tensor_subbuffer(ctx, src1); + vk_subbuffer d_X, d_Y; + + if (qx_needs_dequant) { + d_X = { ctx->prealloc_x, 0, ctx->prealloc_x->size }; + } else { + d_X = d_Qx; + GGML_ASSERT(qx_sz == x_sz); + } + if (qy_needs_dequant || quantize_y) { + d_Y = { ctx->prealloc_y, 0, ctx->prealloc_y->size }; + } else { + d_Y = d_Qy; + } + + if (x_non_contig) { + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); } - // One descriptor set per pipeline (phase A and phase B are now - // separate vk_pipeline objects with different binding layouts, so - // each needs its own descriptor-set request -- unlike the old - // single-shader design's one request(..., 2) for two dispatches of - // the SAME pipeline). - ggml_pipeline_request_descriptor_sets(ctx, batch_pipeline_a, 1); - ggml_pipeline_request_descriptor_sets(ctx, batch_pipeline_b, 1); - if (!ggml_vk_buffer_write_async(compute_ctx, params_ctx->dev_buffer, - params->params_offset, batch_upload.data(), - batch_upload_bytes, true)) { - return false; + GGML_ASSERT(x_sz == ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment)); + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, d_Qx, d_X); + } + if (y_non_contig) { + GGML_ASSERT(y_sz == ggml_type_size(src1->type) * y_ne); + if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || + ctx->prealloc_y_last_tensor_used != src1 || + ctx->prealloc_y_last_k_padded) { + if (ctx->prealloc_y_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, d_Qy, d_Y); + ctx->prealloc_y_last_pipeline_used = to_fp16_vk_1.get(); + ctx->prealloc_y_last_tensor_used = src1; + ctx->prealloc_y_last_k_padded = false; } - compute_ctx->s->buffer->buf.pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer | vk::PipelineStageFlagBits::eHost, - vk::PipelineStageFlagBits::eComputeShader, - {}, - { { vk::AccessFlagBits::eTransferWrite | vk::AccessFlagBits::eHostWrite, - vk::AccessFlagBits::eShaderRead } }, - {}, {}); + } + if (quantize_y) { + if (ctx->prealloc_y_last_pipeline_used != to_q8_1.get() || + ctx->prealloc_y_last_tensor_used != src1 || + ctx->prealloc_y_last_k_padded) { + if (ctx->prealloc_y_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + ggml_vk_quantize_q8_1(ctx, subctx, d_Qy, d_Y, y_ne); + ctx->prealloc_y_last_pipeline_used = to_q8_1.get(); + ctx->prealloc_y_last_tensor_used = src1; + ctx->prealloc_y_last_k_padded = false; + } + } - const vk_subbuffer d_input{io_ctx->dev_buffer, params->input_offset, io_bytes}; - const vk_subbuffer d_output{io_ctx->dev_buffer, params->output_offset, io_bytes}; - const vk_subbuffer d_params{params_ctx->dev_buffer, params->params_offset, batch_upload_bytes}; - // The whole arena, offset 0 -- every selected expert's role data is - // somewhere inside it; the shader computes each block's byte - // address as (this base) + (per-expert offset, from the params - // buffer) + (local byte offset within the quant block), all at byte - // granularity, so no VkDescriptorBufferInfo offset - // alignment applies here (unlike the per-expert path's sub-buffer - // binds, which must satisfy minStorageBufferOffsetAlignment). - const vk_subbuffer d_weights{weight_buffers[0], 0, (VkDeviceSize) weight_sizes[0]}; - const vk_subbuffer d_scratch{ctx->device->wp_fused_batch_scratch, 0, (VkDeviceSize) scratch_bytes}; + // For batch_n, the A matrix is the same for each batch, and B/D use the row stride as the batch stride + uint32_t stride_batch_x = batch_n ? 0 : ne00*ne01; + uint32_t stride_batch_y = batch_n ? ne10 : (ne10*ne11); + uint32_t stride_batch_d = batch_n ? ne20 : (ne20*ne21); - // Placeholder push constants (see the struct comment): neither phase - // reads this value, it exists only so ggml_vk_dispatch_pipeline's - // push_constant_size assertion has something matching what - // ggml_vk_create_pipeline was told each pipeline's layout uses. - const vk_wp_fused_expert_batch_push_constants pc{0}; + if (!ggml_vk_dim01_contiguous(src0) && !qx_needs_dequant) { + stride_batch_x = src0->nb[0] / ggml_type_size(src0->type); + } - // Phase A: gate, up, input, params, scratch (5 bindings, matching - // wp_fused_expert_batch_a.comp's layout(binding=...) declarations - // and the "5" passed to ggml_vk_create_pipeline for - // pipeline_wp_fused_expert_batch_a above). - const uint32_t hgroups = (params->n_ff + 7u) / 8u; - const uint32_t phase_a_wgs = params->n_experts * params->n_tokens * hgroups; - ggml_vk_dispatch_pipeline(ctx, compute_ctx, batch_pipeline_a, - { d_weights, d_weights, d_input, d_params, d_scratch }, - pc, { phase_a_wgs, 1, 1 }); + if (!ggml_vk_dim01_contiguous(src1) && !qy_needs_dequant) { + stride_batch_y = src1->nb[0] / ggml_type_size(src1->type); + } - // Phase B's per-row down-projection reads every expert's Phase A - // output for its token -- a real compute-shader-write -> - // compute-shader-read hazard across the whole scratch buffer, same - // shape as (and using the same helper as) the WP_VK_HOST_READ_FASTPATH - // barrier below. This is now the ONLY inter-phase synchronization: - // phase B's shader itself contains no barrier() (see its top-of-file - // comment) and reads hidden_scratch directly instead of staging - // through shared memory, so this single compute-to-compute buffer - // barrier between the two dispatches is both necessary (phase B's - // reads must not race phase A's writes) and sufficient (there is no - // further intra-phase-B ordering requirement to enforce). - ggml_vk_sync_buffers(ctx, compute_ctx); - - // Phase B: down, output, params, scratch (4 bindings, matching - // wp_fused_expert_batch_b.comp's layout and the "4" passed for - // pipeline_wp_fused_expert_batch_b_q5_1/q8_0 above). - const uint32_t rgroups = (params->n_embd + 7u) / 8u; - const uint32_t phase_b_wgs = params->n_tokens * rgroups; - const vk_subbuffer d_down{weight_buffers[0], 0, (VkDeviceSize) weight_sizes[0]}; - ggml_vk_dispatch_pipeline(ctx, compute_ctx, batch_pipeline_b, - { d_down, d_output, d_params, d_scratch }, - pc, { phase_b_wgs, 1, 1 }); + const uint32_t max_groups_x = ctx->device->properties.limits.maxComputeWorkGroupCount[0]; - n_pipeline_dispatches = 2; - n_workgroups = (uint64_t) phase_a_wgs + phase_b_wgs; - bytes_uploaded_actual = batch_upload_bytes; - } else { - std::vector<uint32_t> upload(total_words, 0); - upload[0] = params->n_tokens; - upload[1] = params->n_embd; - upload[2] = params->n_ff; - upload[3] = params->n_experts; - upload[4] = (uint32_t) gate_blocks; - upload[5] = (uint32_t) down_blocks; - upload[6] = params->add_previous != 0; - std::memcpy(&upload[7], ¶ms->swiglu_limit, sizeof(float)); - upload[8] = (uint32_t) header_words; - for (uint32_t e = 0; e < params->n_experts; ++e) { - for (uint32_t token = 0; token < params->n_tokens; ++token) { - std::memcpy(&upload[upload[8] + (size_t) e * params->n_tokens + token], - ¶ms->route_weights[(size_t) e * params->n_tokens + token], - sizeof(float)); - } - } + uint32_t groups_x = ne01; + uint32_t groups_z = 1; - ggml_pipeline_request_descriptor_sets(ctx, pipeline, params->n_experts); - if (!ggml_vk_buffer_write_async(compute_ctx, params_ctx->dev_buffer, - params->params_offset, upload.data(), - upload_bytes, true)) { - return false; - } - compute_ctx->s->buffer->buf.pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer | vk::PipelineStageFlagBits::eHost, - vk::PipelineStageFlagBits::eComputeShader, - {}, - { { vk::AccessFlagBits::eTransferWrite | vk::AccessFlagBits::eHostWrite, - vk::AccessFlagBits::eShaderRead } }, - {}, {}); - const vk_subbuffer d_input{io_ctx->dev_buffer, params->input_offset, io_bytes}; - const vk_subbuffer d_output{io_ctx->dev_buffer, params->output_offset, io_bytes}; - const vk_subbuffer d_params{params_ctx->dev_buffer, params->params_offset, upload_bytes}; - for (uint32_t e = 0; e < params->n_experts; ++e) { - // Each role is its own sub-buffer at its own byte offset (see the - // comment on ggml_backend_vk_wp_fused_expert_params) -- the shader - // always indexes local block 0 upward within each, so no per-expert - // block-index math is needed on the device side. - const vk_subbuffer d_gate{weight_buffers[e], - (VkDeviceSize) params->gate_byte_offsets[e], (VkDeviceSize) gate_role_bytes}; - const vk_subbuffer d_up{weight_buffers[e], - (VkDeviceSize) params->up_byte_offsets[e], (VkDeviceSize) gate_role_bytes}; - const vk_subbuffer d_down{weight_buffers[e], - (VkDeviceSize) params->down_byte_offsets[e], (VkDeviceSize) down_role_bytes}; - vk_wp_fused_expert_push_constants pc{e}; - ggml_vk_dispatch_pipeline(ctx, compute_ctx, pipeline, - { d_gate, d_up, d_down, d_input, d_output, d_params }, - pc, { params->n_tokens, 1, 1 }); - if (e + 1 < params->n_experts) { - compute_ctx->s->buffer->buf.pipelineBarrier( - vk::PipelineStageFlagBits::eComputeShader, - vk::PipelineStageFlagBits::eComputeShader, - {}, - { { vk::AccessFlagBits::eShaderWrite, - vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite } }, - {}, {}); - } - } - n_pipeline_dispatches = params->n_experts; - n_workgroups = (uint64_t) params->n_experts * params->n_tokens; - bytes_uploaded_actual = upload_bytes; + if (ne01 > max_groups_x) { + groups_z = 64; + groups_x = CEIL_DIV(groups_x, groups_z); } - // *** WP_VK_HOST_READ_FASTPATH -- populate the read-side cache. *** - // - // Mirrors the population half of ggml_backend_vk_graph_compute()'s fastpath - // (see the "submit || last_node" block there): copy this dispatch's output - // range into device->host_read_staging as part of the SAME command buffer - // this dispatch is already recording -- before ctx_end()/submit -- and - // publish host_read_source/offset/size so the next read_result() hits the - // cached staging read instead of a cold device read. Previously this - // function only invalidated the cache (see below) because it writes - // io_ctx->dev_buffer outside graph_compute's own fastpath machinery; that - // made every fused read a real device read (the WP_VK_HOST_READ_FASTPATH - // readback regression). When the fastpath is disabled or ineligible for - // this output, fall through to the same invalidation graph_compute's own - // top-of-function reset performs, so a stale snapshot from a prior graph - // (or a prior fused dispatch whose output didn't qualify) is never served. - { - static const bool host_read_fastpath = [] { - const char * e = getenv("WP_VK_HOST_READ_FASTPATH"); - return e != nullptr && e[0] == '1'; - }(); - static const size_t host_read_fastpath_max = [] { - const char * e = getenv("WP_VK_HOST_READ_FASTPATH_MAX_BYTES"); - return (e != nullptr && e[0] != '\0') ? (size_t) strtoull(e, nullptr, 10) : (size_t) 262144; - }(); - const bool output_host_visible = static_cast<bool>( - io_ctx->dev_buffer->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible); - const bool output_host_cached = static_cast<bool>( - io_ctx->dev_buffer->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached); - if (host_read_fastpath && host_read_fastpath_max != 0 && - output_host_visible && !output_host_cached && io_bytes <= host_read_fastpath_max && - ggml_vk_ensure_host_read_staging_buffer(ctx->device, host_read_fastpath_max)) { - ggml_vk_sync_buffers(ctx, compute_ctx); - compute_ctx->s->buffer->buf.copyBuffer(io_ctx->dev_buffer->buffer, - ctx->device->host_read_staging->buffer, - { { params->output_offset, 0, io_bytes } }); - ctx->device->host_read_source = io_ctx->dev_buffer; - ctx->device->host_read_source_offset = params->output_offset; - ctx->device->host_read_source_size = io_bytes; - } else { - ctx->device->host_read_source.reset(); - ctx->device->host_read_source_offset = 0; - ctx->device->host_read_source_size = 0; - } - } + uint32_t fusion_flags = 0; - vk_command_buffer * cmd_buf = compute_ctx->s != nullptr ? compute_ctx->s->buffer : nullptr; - compute_ctx->submitted_cmd_buf = cmd_buf; - ggml_vk_ctx_end(compute_ctx); - for (auto & copy : compute_ctx->in_memcpys) { - std::memcpy(copy.dst, copy.src, copy.n); - } - compute_ctx->in_memcpys.clear(); - if (submit_async) { - // Attach ctx->fence directly to this dispatch's own submission instead - // of a null-fence submit followed by a second, empty, fence-only - // vkQueueSubmit inside ggml_vk_synchronize()/ggml_backend_vk_synchronize(). - // This is the same WP_VK_FENCE_ON_LAST_SUBMIT trick already applied to - // the normal graph-compute path (see the comment at its call site in - // ggml_vk_compute_forward): a fence signals only after ITS submission - // AND every earlier submission on the (in-order) queue complete, which - // is exactly the guarantee the separate empty submit provided. Every - // fused-expert call is already a single, self-contained submission (one - // command buffer, nothing else queued on this context before the - // caller's synchronize), so the fence can always ride the real submit - // here -- no env-var gate needed, unlike the tiny-graph case upstream. - // Same command buffer, same order, same fence semantics -- one fewer - // queue submission and one fewer wait per request. - // - // Maintain the fence_submitted invariant: never leave an unconsumed - // fence signal behind while submitting more work onto the same fence - // (mirrors the "drain first" check in ggml_vk_compute_forward). - if (ctx->fence_submitted) { - ggml_vk_wait_for_fence(ctx); - ggml_vk_queue_mark_waited(*ctx->device->compute_queue->handle, ctx->fence_epoch); - ctx->fence_submitted = false; - ctx->submit_pending = false; - } - ggml_vk_submit(compute_ctx, ctx->fence); - ctx->fence_submitted = true; - ctx->fence_epoch = ggml_vk_queue_epoch(*ctx->device->compute_queue->handle); - ctx->submit_pending = true; - } else { - ggml_vk_submit(compute_ctx, ctx->fence); - const uint64_t epoch = ggml_vk_queue_epoch(*ctx->device->compute_queue->handle); - ggml_vk_wait_for_fence(ctx); - ggml_vk_queue_mark_waited(*ctx->device->compute_queue->handle, epoch); - ctx->device->device.resetFences({ ctx->fence }); - if (cmd_buf != nullptr) { - cmd_buf->in_use = false; - cmd_buf->buf.reset(); - compute_ctx->submitted_cmd_buf = nullptr; - } - ggml_vk_queue_command_pools_cleanup(ctx->device); - ctx->compute_ctx.reset(); - } - // WP_VK_HOST_READ_FASTPATH cache: populated (or, when ineligible, - // invalidated) above, inside the same command buffer this dispatch - // recorded, before it was ended and submitted. - stats->ns_dispatch = (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>( - std::chrono::steady_clock::now() - started).count(); - stats->n_pipeline_dispatches = n_pipeline_dispatches; - stats->n_fence_waits = submit_async ? 0 : 1; - stats->bytes_uploaded = bytes_uploaded_actual; - stats->n_workgroups = n_workgroups; - return true; -} + vk_subbuffer d_F0 = d_D; + if (ctx->num_additional_fused_ops > 0) { + const ggml_tensor * add = cgraph->nodes[node_idx + 1]; + const ggml_tensor * bias = add->src[0] == dst ? add->src[1] : add->src[0]; -// Take-and-clear, so a publication can never be applied to two nodes. -static bool ggml_vk_wp_take_expert_offsets(vk_wp_expert_offsets & out) { - if (!tls_wp_expert_offsets.valid) { - return false; + d_F0 = ggml_vk_tensor_subbuffer(ctx, bias); + fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; } - out = tls_wp_expert_offsets; - tls_wp_expert_offsets.valid = false; - return true; -} -// Upload one node's per-expert offset array and return the subbuffer to bind. -// -// Each node gets its OWN slice. The dispatches recorded here do not run until -// the graph's command buffer is submitted, so a single shared region written -// eagerly per node would leave every dispatch reading the LAST node's offsets. -// (CUDA sidesteps this because its pointer upload is a stream-ordered async -// memcpy interleaved with the kernels.) The cursor is reset per graph in -// ggml_backend_vk_graph_compute. -static vk_subbuffer ggml_vk_wp_upload_expert_offsets(ggml_backend_vk_context * ctx, - const std::vector<uint32_t> & offsets) { - const size_t nbytes = offsets.size() * sizeof(uint32_t); - const size_t align = std::max<size_t>( - ctx->device->properties.limits.minStorageBufferOffsetAlignment, 4); - const size_t stride = (nbytes + align - 1) / align * align; + vk_subbuffer d_F1 = d_D; + if (ctx->num_additional_fused_ops == 2) { + const ggml_tensor * add = cgraph->nodes[node_idx + 2]; + const ggml_tensor * bias = add->src[0] == cgraph->nodes[node_idx + 1] ? add->src[1] : add->src[0]; - if (ctx->prealloc_wp_expert_off == nullptr || - ctx->prealloc_wp_expert_off->size < ctx->wp_expert_off_cursor + stride) { - const size_t grow = std::max<size_t>( - ctx->wp_expert_off_cursor + stride * 64, 64 * 1024); - vk_buffer old = ctx->prealloc_wp_expert_off; - ctx->prealloc_wp_expert_off = ggml_vk_create_buffer_device(ctx->device, grow); - if (old != nullptr) { - // Anything already recorded still points into `old`; keep it alive - // until the graph completes rather than freeing here. - ctx->wp_expert_off_retired.push_back(old); - } - ctx->wp_expert_off_cursor = 0; + d_F1 = ggml_vk_tensor_subbuffer(ctx, bias); + fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS1; } - const size_t slice = ctx->wp_expert_off_cursor; - ggml_vk_buffer_write(ctx->prealloc_wp_expert_off, slice, offsets.data(), nbytes); - ctx->wp_expert_off_cursor += stride; - return vk_subbuffer{ ctx->prealloc_wp_expert_off, slice, nbytes }; -} + ggml_pipeline_request_descriptor_sets(ctx, dmmv, CEIL_DIV(ne12 * ne13, ctx->device->properties.limits.maxComputeWorkGroupCount[1])); -bool ggml_backend_vk_wp_event_query(void * event) { - auto * wp_event = (ggml_vk_wp_event *) event; - if (wp_event == nullptr) { - return false; - } - return wp_event->device->device.getFenceStatus(wp_event->fence) == vk::Result::eSuccess; -} + uint32_t base_work_group_y = 0; + while (base_work_group_y < ne12 * ne13) { -bool ggml_backend_vk_wp_event_wait(void * event) { - auto * wp_event = (ggml_vk_wp_event *) event; - if (wp_event == nullptr) { - return false; + uint32_t groups_y = std::min((uint32_t)(ne12 * ne13) - base_work_group_y, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + const vk_mat_vec_push_constants pc = { + (uint32_t)ne00, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne01, + stride_batch_x, stride_batch_y, stride_batch_d, + fusion_flags, base_work_group_y, + (uint32_t)ne02, (uint32_t)ne12, (uint32_t)r2, (uint32_t)r3, + }; + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, + { + d_X, + d_Y, + d_D, + d_F0, + d_F1, + }, + pc, { groups_x, groups_y, groups_z }); + base_work_group_y += groups_y; } - std::lock_guard<std::recursive_mutex> guard(wp_event->device->mutex); - VK_CHECK(wp_event->device->device.waitForFences({ wp_event->fence }, true, UINT64_MAX), "wp stage_in waitForFences", wp_event->device); - ggml_vk_queue_command_pools_cleanup(wp_event->device); - return true; -} -void ggml_backend_vk_wp_event_free(void * event) { - auto * wp_event = (ggml_vk_wp_event *) event; - if (wp_event == nullptr) { - return; + if (x_non_contig) { + ctx->prealloc_x_need_sync = true; + } + if (y_non_contig || quantize_y) { + ctx->prealloc_y_need_sync = true; } - ggml_backend_vk_wp_event_wait(wp_event); - wp_event->device->device.destroyFence(wp_event->fence); - delete wp_event; } -static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, uint32_t n, uint32_t k, bool disable_split_k, const vk_pipeline& pipeline) { - VK_LOG_DEBUG("ggml_vk_guess_split_k(" << m << ", " << n << ", " << k << ", " << disable_split_k << ")"); +static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { + ggml_tensor * dst = cgraph->nodes[node_idx]; + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + VK_LOG_DEBUG("ggml_vk_mul_mat_p021_f16_f32(" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; + std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; + std::cerr << "))"); + GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1)); + GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // NOLINT + GGML_ASSERT(src1->nb[0] <= src1->nb[1] && src1->nb[2] <= src1->nb[3]); // NOLINT + GGML_ASSERT(src0->type == GGML_TYPE_F16); + GGML_ASSERT(src1->type == GGML_TYPE_F32); - if (disable_split_k) { - return 1; - } + const uint64_t ne00 = src0->ne[0]; + const uint64_t ne01 = src0->ne[1]; + const uint64_t ne02 = src0->ne[2]; + // const uint64_t ne03 = src0->ne[3]; - uint32_t split_k = 1; - if (ctx->device->shader_core_count != 0 && m >= pipeline->wg_denoms[0] && n >= pipeline->wg_denoms[1]) { - // If k is 'large' and the SMs will fill less than halfway, use split_k. - uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]); - uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]); + //const uint64_t ne10 = src1->ne[0]; + const uint64_t ne11 = src1->ne[1]; + const uint64_t ne12 = src1->ne[2]; + // const uint64_t ne13 = src1->ne[3]; - if (k >= 2048) { - if (m_tiles * n_tiles <= ctx->device->shader_core_count / 2) { - split_k = ctx->device->shader_core_count / (m_tiles * n_tiles); - } else if (m_tiles * n_tiles <= ctx->device->shader_core_count * 2 / 3) { - split_k = 3; - } - // Cap the split at 8x. Unless k is huge this is a lot of overhead. - split_k = std::min(split_k, 8u); + GGML_ASSERT(ne11 == 1); - // ggml_vk_matmul will align the splits to be a multiple of 256. - // If this rounded up size would cause the last split to be empty, - // then reduce the split count. - while (true) { - if (split_k == 1) { - break; - } - uint32_t k_split = CEIL_DIV(k, split_k); - k_split = ROUNDUP_POW2(k_split, 256); - if (k_split * (split_k - 1) < k) { - break; - } - split_k--; - } - } + // With grouped query attention there are > 1 Q matrices per K, V matrix. + uint32_t gqa_ratio = (uint32_t)ne12 / (uint32_t)ne02; + if (gqa_ratio > 8 || gqa_ratio == 0 || ne12 != ne02 * gqa_ratio) { + gqa_ratio = 1; } - return split_k; -} + vk_pipeline pipeline = ctx->device->pipeline_mul_mat_vec_p021_f16_f32[gqa_ratio - 1]; + + if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { + pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); + } -static vk_pipeline ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, uint32_t m, uint32_t n, bool aligned, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); + { + // Request descriptor sets + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + } - // The q8_1 (integer dot) mmq path uses a different shader with its own - // shared-memory layout, so use the int-specific availability flags. - const bool is_q8_1 = (src1_type == GGML_TYPE_Q8_1); - const bool mm_l = is_q8_1 ? ctx->device->mul_mat_l_int[src0_type] : ctx->device->mul_mat_l[src0_type]; - const bool mm_m = is_q8_1 ? ctx->device->mul_mat_m_int[src0_type] : ctx->device->mul_mat_m[src0_type]; - const bool mm_s = is_q8_1 ? ctx->device->mul_mat_s_int[src0_type] : ctx->device->mul_mat_s[src0_type]; + vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, cgraph->nodes[node_idx + ctx->num_additional_fused_ops], true); + vk_subbuffer d_Qx = ggml_vk_tensor_subbuffer(ctx, src0); + vk_subbuffer d_Qy = ggml_vk_tensor_subbuffer(ctx, src1, true); - if (ctx->device->coopmat2) { - const uint32_t shader_core_count = ctx->device->shader_core_count; - const uint32_t tiles_l = CEIL_DIV(m, mmp->a_l->wg_denoms[0]) * CEIL_DIV(n, mmp->a_l->wg_denoms[1]); - const uint32_t tiles_m = CEIL_DIV(m, mmp->a_m->wg_denoms[0]) * CEIL_DIV(n, mmp->a_m->wg_denoms[1]); + vk_subbuffer d_F0 = d_D; - // Use large shader when the N dimension is greater than the medium shader's tile size - uint32_t crossover_large = mmp->m->wg_denoms[1]; + uint32_t fusion_flags = 0; - // Prefer large over medium if either: - // - medium or large tiles would overfill the GPU - // - large tiles with a split_k==3 fits in the GPU and medium tiles with split_k==2 does not - // (medium with split_k==2 is probably better if it fits - more workgroups running and less split_k overhead) - bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count || - // split_k==3 with large tiles likely better than medium tiles with no split_k. - (tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2); + if (ctx->num_additional_fused_ops > 0) { + const ggml_tensor * add = cgraph->nodes[node_idx + 1]; + const ggml_tensor * bias = add->src[0] == dst ? add->src[1] : add->src[0]; - if ((mm_l && (n > crossover_large && prefer_large)) || (!mm_m && !mm_s)) { - return aligned ? mmp->a_l : mmp->l; - } - // Use medium shader when the N dimension is greater than the small shader's tile size - uint32_t crossover_medium = mmp->s->wg_denoms[1]; - if ((mm_m && (n > crossover_medium)) || !mm_s) { - return aligned ? mmp->a_m : mmp->m; - } - return aligned ? mmp->a_s : mmp->s; + d_F0 = ggml_vk_tensor_subbuffer(ctx, bias); + fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; } - if ((mm_s && (m <= 32 || n <= 32)) || (!mm_m && !mm_l)) { - return aligned ? mmp->a_s : mmp->s; - } - if ((mm_m && (m <= 64 || n <= 64)) || !mm_l) { - return aligned ? mmp->a_m : mmp->m; + vk_subbuffer d_F1 = d_D; + if (ctx->num_additional_fused_ops > 1) { + const ggml_tensor * bias = cgraph->nodes[node_idx + 2]->src[1]; + + d_F1 = ggml_vk_tensor_subbuffer(ctx, bias); + fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS1; } - return aligned ? mmp->a_l : mmp->l; -} -static uint32_t ggml_vk_guess_matmul_pipeline_align(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline_align(" << m << ", " << n << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); - return ggml_vk_guess_matmul_pipeline(ctx, mmp, m, n, true, src0_type, src1_type)->align; -} + // compute -static void ggml_vk_matmul( - ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, - vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, - uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, - uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, - uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, - uint32_t padded_n) { - VK_LOG_DEBUG("ggml_vk_matmul(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << (split_k_buffer.buffer != nullptr ? split_k_buffer.buffer->buffer : VK_NULL_HANDLE) << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ", padded_n: " << padded_n << ")"); - if (split_k == 1) { - ggml_pipeline_request_descriptor_sets(ctx, pipeline, CEIL_DIV(batch, ctx->device->properties.limits.maxComputeWorkGroupCount[2])); + vk_mat_vec_p021_push_constants pc = { + (uint32_t)ne00, (uint32_t)ne01, (uint32_t)ne02, (uint32_t)ne12, + 0, 0, fusion_flags + }; - uint32_t base_work_group_z = 0; - while (base_work_group_z < batch) { - uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]); + init_pushconst_tensor_offsets(ctx, pc, src0, src1, nullptr, nullptr, cgraph->nodes[node_idx + ctx->num_additional_fused_ops]); - const vk_mat_mat_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k, ne02, ne12, broadcast2, broadcast3, padded_n }; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d }, pc, { m, n, groups_z }); - base_work_group_z += groups_z; - } - return; + uint32_t workgroups_z = (uint32_t)ne12; + // When gqa_ratio > 1, each invocation does multiple rows and we can launch fewer workgroups + if (gqa_ratio > 1) { + workgroups_z /= gqa_ratio; } - if (ctx->prealloc_split_k_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + d_Qx, + d_Qy, + d_D, + d_F0, + d_F1, + }, pc, { 1, (uint32_t)ne01, workgroups_z }); +} - GGML_ASSERT(batch_stride_d == m * n); +static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { + ggml_tensor * dst = cgraph->nodes[node_idx]; + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + VK_LOG_DEBUG("ggml_vk_mul_mat_nc_f16_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; + std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; + std::cerr << "))"); + GGML_ASSERT(!ggml_is_transposed(src0)); + GGML_ASSERT(!ggml_is_transposed(src1)); + GGML_ASSERT(!ggml_is_permuted(src0)); + GGML_ASSERT(src0->type == GGML_TYPE_F16); + GGML_ASSERT(src1->type == GGML_TYPE_F32); - // Round the split size up to a multiple of 256 (k-quant alignment) - uint32_t k_split = CEIL_DIV(k, split_k); - k_split = ROUNDUP_POW2(k_split, 256); + const uint64_t ne00 = src0->ne[0]; + const uint64_t ne01 = src0->ne[1]; + const uint64_t ne02 = src0->ne[2]; + const uint64_t ne03 = src0->ne[3]; - ggml_pipeline_request_descriptor_sets(ctx, pipeline, CEIL_DIV(batch, ctx->device->properties.limits.maxComputeWorkGroupCount[2])); + const uint64_t nb01 = src0->nb[1]; + const uint64_t nb02 = src0->nb[2]; - uint32_t base_work_group_z = 0; - while (base_work_group_z < batch) { - uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]); + const uint64_t nb12 = src1->nb[2]; - const vk_mat_mat_push_constants pc1 = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k_split, ne02, ne12, broadcast2, broadcast3, padded_n }; - // Make sure enough workgroups get assigned for split k to work - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer }, pc1, { (CEIL_DIV(m, pipeline->wg_denoms[0]) * pipeline->wg_denoms[0]) * split_k, n, groups_z }); - base_work_group_z += groups_z; - } - ggml_vk_sync_buffers(ctx, subctx); - const std::array<uint32_t, 2> pc2 = { (uint32_t)(m * n * batch), split_k }; - ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_matmul_split_k_reduce, { split_k_buffer, d }, pc2, { m * n * batch, 1, 1 }); - ctx->prealloc_split_k_need_sync = true; -} + // const uint64_t ne10 = src1->ne[0]; + const uint64_t ne11 = src1->ne[1]; + const uint64_t ne12 = src1->ne[2]; + // const uint64_t ne13 = src1->ne[3]; -static vk_pipeline ggml_vk_guess_matmul_id_pipeline(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, uint32_t m, uint32_t n, bool aligned, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_id_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); + const uint32_t nb03 = (uint32_t)(src0->nb[3] / sizeof(ggml_fp16_t)); + const uint32_t nb13 = (uint32_t)(src1->nb[3] / sizeof(float)); + const uint32_t nb23 = (uint32_t)(dst->nb[3] / sizeof(float)); - // The q8_1 (integer dot) mmq path uses a different shader with its own - // shared-memory layout, so use the int-specific availability flags. - const bool is_q8_1 = (src1_type == GGML_TYPE_Q8_1); - const bool mm_l = is_q8_1 ? ctx->device->mul_mat_id_l_int[src0_type] : ctx->device->mul_mat_id_l[src0_type]; - const bool mm_m = is_q8_1 ? ctx->device->mul_mat_id_m_int[src0_type] : ctx->device->mul_mat_id_m[src0_type]; - const bool mm_s = is_q8_1 ? ctx->device->mul_mat_id_s_int[src0_type] : ctx->device->mul_mat_id_s[src0_type]; + GGML_ASSERT(ne11 == 1); + GGML_ASSERT(src0->ne[3] == src1->ne[3]); // checked in supports_op - if (ctx->device->coopmat2) { - // Use large shader when the N dimension is greater than the medium shader's tile size - uint32_t crossover_large = mmp->m->wg_denoms[1]; - if ((mm_l && (n > crossover_large)) || (!mm_m && !mm_s)) { - return aligned ? mmp->a_l : mmp->l; - } - // Use medium shader when the N dimension is greater than the small shader's tile size - uint32_t crossover_medium = mmp->s->wg_denoms[1]; - if ((mm_m && (n > crossover_medium)) || !mm_s) { - return aligned ? mmp->a_m : mmp->m; - } - return aligned ? mmp->a_s : mmp->s; - } + const uint32_t row_stride_x = nb01 / sizeof(ggml_fp16_t); + const uint32_t channel_stride_x = nb02 / sizeof(ggml_fp16_t); + const uint32_t channel_stride_y = nb12 / sizeof(float); - if ((mm_s && (m <= 32 || n <= 32)) || (!mm_m && !mm_l)) { - return aligned ? mmp->a_s : mmp->s; + vk_pipeline pipeline = ctx->device->pipeline_mul_mat_vec_nc_f16_f32; + if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { + pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); } - if ((mm_m && (m <= 64 || n <= 64)) || !mm_l) { - return aligned ? mmp->a_m : mmp->m; + + { + // Request descriptor sets + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); } - return aligned ? mmp->a_l : mmp->l; -} -static uint32_t ggml_vk_guess_matmul_id_pipeline_align(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline_align(" << m << ", " << n << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); - return ggml_vk_guess_matmul_id_pipeline(ctx, mmp, m, n, true, src0_type, src1_type)->align; -} + vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, cgraph->nodes[node_idx + ctx->num_additional_fused_ops], true); + vk_subbuffer d_Qx = ggml_vk_tensor_subbuffer(ctx, src0); + vk_subbuffer d_Qy = ggml_vk_tensor_subbuffer(ctx, src1, true); + vk_subbuffer d_F0 = d_D; -static void ggml_vk_matmul_id( - ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, - vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& ids, const vk_subbuffer & expert_count_buf, - const vk_subbuffer & wp_off_buf, - uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, - uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, - uint32_t n_as, uint32_t nei0, uint32_t nei1, uint32_t nbi1, uint32_t ne11, - bool hoist_row_ids, uint32_t paged) { - VK_LOG_DEBUG("ggml_vk_matmul_id(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), ids: (" << ids.buffer->buffer << ", " << ids.offset << ", " << ids.size << "), expert_count: (" << expert_count_buf.buffer->buffer << ", " << expert_count_buf.offset << ", " << expert_count_buf.size << "), " << - "m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", " << - "batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", " << - "n_as: " << n_as << ", nei0: " << nei0 << ", nei1: " << nei1 << ", nbi1: " << nbi1 << ", ne11: " << ne11 << ")"); - const vk_mat_mat_id_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, - nei0, nei1, nbi1, ne11, n_as, uint32_t(hoist_row_ids), paged }; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf, wp_off_buf }, pc, { m, nei1, n_as }); -} + uint32_t fusion_flags = 0; -static bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) { - return - tensor->nb[0] == ggml_type_size(tensor->type) && - tensor->nb[1] == (tensor->nb[0]*tensor->ne[0])/ggml_blck_size(tensor->type) && - (tensor->ne[3] == 1 || tensor->nb[3] == tensor->nb[2]*tensor->ne[2]); -} - -static vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to) { + if (ctx->num_additional_fused_ops > 0) { + const ggml_tensor * add = cgraph->nodes[node_idx + 1]; + const ggml_tensor * bias = add->src[0] == dst ? add->src[1] : add->src[0]; - // Choose "contiguous copy" shader if src/dst are contiguous - bool contig = ggml_is_contiguous(src) && (!dst || ggml_is_contiguous(dst)); + d_F0 = ggml_vk_tensor_subbuffer(ctx, bias); + fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; + } - // Use optimized "transpose" shader if src dim1 is the innermost dimension. - bool transpose = dst && src->nb[1] == ggml_type_size(to) && ggml_are_same_shape(dst, src); + vk_subbuffer d_F1 = d_D; + if (ctx->num_additional_fused_ops > 1) { + const ggml_tensor * bias = cgraph->nodes[node_idx + 2]->src[1]; - if (transpose && src->type == to) { - if (ggml_type_size(to) == 4) { - return ctx->device->pipeline_cpy_transpose_32; - } else if (ggml_type_size(to) == 2) { - return ctx->device->pipeline_cpy_transpose_16; - } + d_F1 = ggml_vk_tensor_subbuffer(ctx, bias); + fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS1; } - // Same, for a 0<->2 swap: src dim2 is the innermost dimension. - bool transpose02 = dst && !contig && src->nb[2] == ggml_type_size(to) && - ggml_is_contiguous(dst) && ggml_are_same_shape(dst, src); + // compute + vk_mat_vec_nc_push_constants pc = { + (uint32_t)ne00, (uint32_t)ne01, + row_stride_x, channel_stride_x, channel_stride_y, + (uint32_t)(ne12 / ne02), (uint32_t)ne12, + 0, 0, + nb03, nb13, nb23, fusion_flags + }; - if (transpose02 && src->type == to) { - if (ggml_type_size(to) == 4) { - return ctx->device->pipeline_cpy_transpose_02_32; - } else if (ggml_type_size(to) == 2) { - return ctx->device->pipeline_cpy_transpose_02_16; - } - } + init_pushconst_tensor_offsets(ctx, pc, src0, src1, nullptr, nullptr, cgraph->nodes[node_idx + ctx->num_additional_fused_ops]); - if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_F32) { - if (contig) { - return ctx->device->pipeline_contig_cpy_f32_f32; - } else { - return ctx->device->pipeline_cpy_f32_f32; - } - } - if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_F16) { - if (contig) { - return ctx->device->pipeline_contig_cpy_f32_f16; - } else { - return ctx->device->pipeline_cpy_f32_f16; - } - } - if (src->type == GGML_TYPE_F16 && to == GGML_TYPE_F16) { - if (contig) { - return ctx->device->pipeline_contig_cpy_f16_f16; - } else { - return ctx->device->pipeline_cpy_f16_f16; - } - } - if (src->type == GGML_TYPE_F16 && to == GGML_TYPE_F32) { - if (contig) { - return ctx->device->pipeline_contig_cpy_f16_f32; - } else { - return ctx->device->pipeline_cpy_f16_f32; - } - } - if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_BF16) { - if (contig) { - return ctx->device->pipeline_contig_cpy_f32_bf16; - } else { - return ctx->device->pipeline_cpy_f32_bf16; - } - } - if (src->type == GGML_TYPE_BF16 && to == GGML_TYPE_F32) { - if (contig) { - return ctx->device->pipeline_contig_cpy_bf16_f32; - } else { - return ctx->device->pipeline_cpy_bf16_f32; - } + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + d_Qx, + d_Qy, + d_D, + d_F0, + d_F1, + }, pc, { (uint32_t)ne03, (uint32_t)ne01, (uint32_t)ne12 }); +} + +static int ggml_vk_fwht_pipeline_idx(int64_t n) { + switch (n) { + case 64: return 0; + case 128: return 1; + case 256: return 2; + case 512: return 3; + default: return -1; } - if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_I32) { - if (contig) { - return ctx->device->pipeline_contig_cpy_f32_i32; - } else { - return ctx->device->pipeline_cpy_f32_i32; - } +} + +bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor * src1, const ggml_tensor * dst) { + if (ctx->num_additional_fused_ops != 0) { + return false; } - if (src->type == GGML_TYPE_I32 && to == GGML_TYPE_F32) { - if (contig) { - return ctx->device->pipeline_contig_cpy_i32_f32; - } else { - return ctx->device->pipeline_cpy_i32_f32; - } + + if (ggml_get_op_params_i32(dst, 1) != GGML_HINT_SRC0_IS_HADAMARD) { + return false; } - if (src->type == GGML_TYPE_F32) { - switch (to) { - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_TURBO4_0: - return ctx->device->pipeline_cpy_f32_quant[to]; - default: - break; - } + + const int idx = ggml_vk_fwht_pipeline_idx(src1->ne[0]); + if (idx < 0 || ctx->device->pipeline_fwht_f32[idx] == nullptr) { + return false; } - if (to == GGML_TYPE_F32) { - switch (src->type) { - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_IQ4_NL: - return ctx->device->pipeline_cpy_quant_f32[src->type]; - default: - break; - } + if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; } - if (src->type == to) { - // Copy two or four bytes at a time, depending on block size. - // For quantized types, we scale by block size/type size. But - // this path is also used for bf16->bf16 for example, where the - // type size must be exactly 2 or 4. - GGML_ASSERT(ggml_is_quantized(to) || ggml_type_size(src->type) == 2 || ggml_type_size(src->type) == 4); - if ((ggml_type_size(src->type) % 4) == 0) { - if (contig) { - return ctx->device->pipeline_contig_cpy_f32_f32; - } else { - return ctx->device->pipeline_cpy_f32_f32; - } - } else { - if (contig) { - return ctx->device->pipeline_contig_cpy_f16_f16; - } else { - return ctx->device->pipeline_cpy_f16_f16; - } - } + if (!ggml_is_contiguous(src1)) { + return false; } + GGML_ASSERT(ggml_is_contiguous(dst)); - std::cerr << "Missing CPY op for types: " << ggml_type_name(src->type) << " " << ggml_type_name(to) << std::endl; - GGML_ABORT("fatal error"); + return true; } -static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, const vk_subbuffer & in, const vk_subbuffer & out) { - VK_LOG_DEBUG("ggml_vk_cpy_to_contiguous((" << tensor << ", type=" << tensor->type << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), "; - std::cerr << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ")"); - - const uint32_t ne = ggml_nelements(tensor); - std::array<uint32_t, 3> elements; - - if (ne > 262144) { - elements = { 512, 512, CEIL_DIV(ne, 262144) }; - } else if (ne > 512) { - elements = { 512, CEIL_DIV(ne, 512), 1 }; - } else { - elements = { ne, 1, 1 }; - } +void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst) { + const int idx = ggml_vk_fwht_pipeline_idx(src->ne[0]); + vk_pipeline pipeline = ctx->device->pipeline_fwht_f32[idx]; - vk_op_unary_push_constants pc = vk_op_unary_push_constants_init(tensor, tensor, ne); - pc.nb10 = 1; - pc.nb11 = (uint32_t)tensor->ne[0]; - pc.nb12 = (uint32_t)(tensor->ne[0] * tensor->ne[1]); - pc.nb13 = (uint32_t)(tensor->ne[0] * tensor->ne[1] * tensor->ne[2]); - init_pushconst_fastdiv(pc); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { in, out }, pc, elements); - ggml_vk_sync_buffers(ctx, subctx); -} + const uint32_t rows_per_workgroup = 4; + const uint32_t n_rows = (uint32_t)ggml_nrows(src); + const uint32_t max_workgroups_x = ctx->device->properties.limits.maxComputeWorkGroupCount[0]; -// Copy/convert tensor into a caller-defined dense layout. Destination strides -// are in output elements, not bytes. -static void ggml_vk_cpy_to_strided( - ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, - const vk_subbuffer & in, const vk_subbuffer & out, - uint32_t nb10, uint32_t nb11, uint32_t nb12, uint32_t nb13) { - VK_LOG_DEBUG("ggml_vk_cpy_to_strided((" << tensor << ", type=" << tensor->type << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), "; - std::cerr << "dst_nb=(" << nb10 << ", " << nb11 << ", " << nb12 << ", " << nb13 << "), buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ")"); + const uint32_t total_workgroups = CEIL_DIV(n_rows, rows_per_workgroup); + const uint32_t workgroups_x = std::min(total_workgroups, max_workgroups_x); + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - const uint32_t ne = ggml_nelements(tensor); - std::array<uint32_t, 3> elements; + const vk_subbuffer src_buf = ggml_vk_tensor_subbuffer(ctx, src, true); + const vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); - if (ne > 262144) { - elements = { 512, 512, CEIL_DIV(ne, 262144) }; - } else if (ne > 512) { - elements = { 512, CEIL_DIV(ne, 512), 1 }; - } else { - elements = { ne, 1, 1 }; - } + vk_op_fwht_push_constants pc = { + n_rows, + 0, + 0, + 1.0f / std::sqrt((float)src->ne[0]), + }; + init_pushconst_tensor_offsets(ctx, pc, src, nullptr, nullptr, nullptr, dst); - vk_op_unary_push_constants pc = vk_op_unary_push_constants_init(tensor, tensor, ne); - pc.nb10 = nb10; - pc.nb11 = nb11; - pc.nb12 = nb12; - pc.nb13 = nb13; - init_pushconst_fastdiv(pc); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { in, out }, pc, elements); - ggml_vk_sync_buffers(ctx, subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src_buf, dst_buf }, pc, { workgroups_x, 1, 1 }); } -static vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type) { - switch(type) { - case GGML_TYPE_Q8_1: - return ctx->device->pipeline_quantize_q8_1_x4; - default: - std::cerr << "Missing quantize pipeline for type: " << ggml_type_name(type) << std::endl; - GGML_ABORT("fatal error"); - } +static uint32_t ggml_vk_nb_elem(const ggml_tensor * t, int i) { + return (uint32_t)(t->nb[i] / ggml_type_size(t->type)); } -static void ggml_vk_quantize_q8_1(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_subbuffer & in, const vk_subbuffer & out, uint32_t ne) { - VK_LOG_DEBUG("ggml_vk_quantize_q8_1(" << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ", " << ne << ")"); +void ggml_vk_dsv4_hc_comb(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * mixes, const ggml_tensor * scale, const ggml_tensor * base, ggml_tensor * dst) { + VK_LOG_DEBUG("ggml_vk_dsv4_hc_comb(" << mixes << ", " << scale << ", " << base << ", " << dst << ")"); - vk_pipeline pipeline = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); + vk_pipeline pipeline = ctx->device->pipeline_dsv4_hc_comb_f32; + GGML_ASSERT(pipeline != nullptr); - const uint32_t num_blocks = CEIL_DIV(ne, pipeline->wg_denoms[0]); - // clamp the number of elements to the max workgroup count. The shader will iterate over the total number of blocks. - const uint64_t max_elements = std::min<uint64_t>(uint64_t{ctx->device->properties.limits.maxComputeWorkGroupCount[0]} * pipeline->wg_denoms[0], std::numeric_limits<uint32_t>::max()); - const uint32_t elements = std::min(ne, static_cast<uint32_t>(max_elements)); + const uint32_t n_tokens = (uint32_t)mixes->ne[1]; - const vk_quantize_q8_1_push_constants pc = { - ne, - num_blocks, + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + + const vk_subbuffer mixes_buf = ggml_vk_tensor_subbuffer(ctx, mixes, true); + const vk_subbuffer scale_buf = ggml_vk_tensor_subbuffer(ctx, scale, true); + const vk_subbuffer base_buf = ggml_vk_tensor_subbuffer(ctx, base, true); + const vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); + + vk_op_dsv4_hc_comb_push_constants pc = { + n_tokens, + ggml_vk_nb_elem(mixes, 0), ggml_vk_nb_elem(mixes, 1), + ggml_vk_nb_elem(scale, 0), + ggml_vk_nb_elem(base, 0), + ggml_vk_nb_elem(dst, 0), ggml_vk_nb_elem(dst, 1), ggml_vk_nb_elem(dst, 2), + 0, 0, 0, 0, + ggml_get_op_params_f32(dst, 0), + (uint32_t)ggml_get_op_params_i32(dst, 1), }; + init_pushconst_tensor_offsets(ctx, pc, mixes, scale, base, nullptr, dst); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { in, out }, pc, { elements, 1, 1 }); - ggml_vk_sync_buffers(ctx, subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { mixes_buf, scale_buf, base_buf, dst_buf }, pc, { n_tokens, 1, 1 }); } -static vk_pipeline ggml_vk_get_64b_indexing_pipeline(ggml_backend_vk_context * ctx, vk_pipeline &pipeline) { - GGML_UNUSED(ctx); -#if defined(VK_EXT_shader_64bit_indexing) - vk_pipeline *ptr = &pipeline; - while (*ptr) { - if ((*ptr)->is_64b_indexing) { - return *ptr; - } - ptr = &(*ptr)->next; - } -#endif - return pipeline; -} +void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * weights, ggml_tensor * dst) { + VK_LOG_DEBUG("ggml_vk_dsv4_hc_pre(" << x << ", " << weights << ", " << dst << ")"); -static constexpr uint32_t GGML_VK_MUL_MAT_FORCE_MM_REFERENCE_TOKENS = 512; -// pinned MUL_MAT default: the mat-vec shader in fixed groups of this many columns -static constexpr uint32_t GGML_VK_MUL_MAT_PIN_GROUP = 8; + const float scale = ggml_get_op_params_f32(dst, 0); + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; -static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool disable_split_k, bool force_mm) { - VK_LOG_DEBUG("ggml_vk_mul_mat_q_f16((" << src0 << ", name=" << src0->name << ", type=" << ggml_type_name(src0->type) << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; - std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << ggml_type_name(src1->type) << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << ggml_type_name(dst->type) << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; - std::cerr << "))"); - GGML_ASSERT(ggml_vk_dim01_contiguous(src0) || src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); // NOLINT - GGML_ASSERT(ggml_vk_dim01_contiguous(src1) || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); // NOLINT + vk_pipeline pipeline = gated ? ctx->device->pipeline_dsv4_hc_pre_gated_f32 : ctx->device->pipeline_dsv4_hc_pre_f32; + GGML_ASSERT(pipeline != nullptr); - const uint64_t ne00 = src0->ne[0]; - const uint64_t ne01 = src0->ne[1]; - const uint64_t ne02 = src0->ne[2]; - const uint64_t ne03 = src0->ne[3]; + const uint32_t n_embd = (uint32_t)x->ne[0]; + const uint32_t n_tokens = (uint32_t)x->ne[2]; - const uint64_t ne10 = src1->ne[0]; - const uint64_t ne11 = src1->ne[1]; - const uint64_t ne12 = src1->ne[2]; - const uint64_t ne13 = src1->ne[3]; + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - const uint64_t ne21 = dst->ne[1]; - const uint32_t stride_d = dst->nb[1] / ggml_type_size(dst->type); - const uint32_t stride_batch_d = stride_d*ne21; + const vk_subbuffer x_buf = ggml_vk_tensor_subbuffer(ctx, x, true); + const vk_subbuffer w_buf = ggml_vk_tensor_subbuffer(ctx, weights, true); + const vk_subbuffer d_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); - const uint64_t r2 = ne12 / ne02; - const uint64_t r3 = ne13 / ne03; + vk_op_dsv4_hc_pre_push_constants pc = { + n_embd, n_tokens, + ggml_vk_nb_elem(x, 0), ggml_vk_nb_elem(x, 1), ggml_vk_nb_elem(x, 2), + ggml_vk_nb_elem(weights, 0), ggml_vk_nb_elem(weights, 1), ggml_vk_nb_elem(weights, 2), + ggml_vk_nb_elem(dst, 0), ggml_vk_nb_elem(dst, 1), + 0, 0, 0, + scale, + }; + init_pushconst_tensor_offsets(ctx, pc, x, weights, nullptr, nullptr, dst); - // Reformat and convert to fp16 if non-contiguous, or for coopmat2 for better perf - const bool x_non_contig = (ctx->device->coopmat2 && src0->type == GGML_TYPE_F32) || - !ggml_vk_dim01_contiguous(src0); - const bool y_non_contig = (ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) || - (src0->type == GGML_TYPE_BF16 && src1->type != GGML_TYPE_BF16) || - !ggml_vk_dim01_contiguous(src1); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, w_buf, d_buf }, pc, { n_embd, n_tokens, 1 }); +} - // If src0 is BF16, try to use a BF16 x BF16 multiply - ggml_type f16_type = src0->type == GGML_TYPE_BF16 ? GGML_TYPE_BF16 : GGML_TYPE_F16; +void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * residual, const ggml_tensor * post, const ggml_tensor * comb, ggml_tensor * dst) { + VK_LOG_DEBUG("ggml_vk_dsv4_hc_post(" << x << ", " << residual << ", " << post << ", " << comb << ", " << dst << ")"); - const bool y_f32_kernel = src1->type == GGML_TYPE_F32 && !y_non_contig; + vk_pipeline pipeline = comb ? ctx->device->pipeline_dsv4_hc_post_f32 : ctx->device->pipeline_dsv4_hc_post_nocomb_f32; + GGML_ASSERT(pipeline != nullptr); - bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0; + const uint32_t n_embd = (uint32_t)x->ne[0]; + const uint32_t n_tokens = (uint32_t)x->ne[1]; - // Check for mmq first - vk_matmul_pipeline mmp = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr; + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - if (mmp == nullptr) { - // Fall back to f16 dequant mul mat - mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]); - quantize_y = false; - } + const vk_subbuffer x_buf = ggml_vk_tensor_subbuffer(ctx, x, true); + const vk_subbuffer r_buf = ggml_vk_tensor_subbuffer(ctx, residual, true); + const vk_subbuffer p_buf = ggml_vk_tensor_subbuffer(ctx, post, true); + const vk_subbuffer c_buf = comb ? ggml_vk_tensor_subbuffer(ctx, comb, true) : x_buf; + const vk_subbuffer d_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); + + vk_op_dsv4_hc_post_push_constants pc = { + n_embd, n_tokens, + ggml_vk_nb_elem(x, 0), ggml_vk_nb_elem(x, 1), + ggml_vk_nb_elem(residual, 0), ggml_vk_nb_elem(residual, 1), ggml_vk_nb_elem(residual, 2), + ggml_vk_nb_elem(post, 0), ggml_vk_nb_elem(post, 1), + comb ? ggml_vk_nb_elem(comb, 0) : 0, comb ? ggml_vk_nb_elem(comb, 1) : 0, comb ? ggml_vk_nb_elem(comb, 2) : 0, + ggml_vk_nb_elem(dst, 0), ggml_vk_nb_elem(dst, 1), ggml_vk_nb_elem(dst, 2), + 0, 0, 0, 0, 0, + }; + init_pushconst_tensor_offsets(ctx, pc, x, residual, post, comb, dst); - const bool qx_needs_dequant = mmp == nullptr || x_non_contig; - const bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, r_buf, p_buf, c_buf, d_buf }, pc, { n_embd, n_tokens, 1 }); +} - if (qx_needs_dequant) { - // Fall back to dequant + f16 mulmat - mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]); +static void ggml_vk_turbo_wht(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; // InnerQ scale_inv (may be nullptr) + + int direction, group_size; + memcpy(&direction, dst->op_params + 0, sizeof(int)); + memcpy(&group_size, dst->op_params + sizeof(int), sizeof(int)); + + const int64_t head_dim = src0->ne[0]; + const int64_t n_heads = ggml_nelements(src0) / head_dim; + const int64_t groups_per_head = head_dim / group_size; + const int64_t n_groups = groups_per_head * n_heads; + + if (n_groups == 0) { + return; } - // Not implemented - GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT + vk_pipeline pipeline = ctx->device->pipeline_turbo_wht_f32; - const ggml_type effective_src1_type = quantize_y ? GGML_TYPE_Q8_1 : (y_f32_kernel ? GGML_TYPE_F32 : src1->type); - // GGML_VK_PIN_REF_TOKENS overrides the pinned reference column count; 0 = follow the real ne11 - static const uint32_t pin_ref_tokens = [] { - const char * env = std::getenv("GGML_VK_PIN_REF_TOKENS"); - if (env == nullptr || env[0] == '\0') { - return GGML_VK_MUL_MAT_FORCE_MM_REFERENCE_TOKENS; - } - const long value = std::strtol(env, nullptr, 10); - return value < 0 ? GGML_VK_MUL_MAT_FORCE_MM_REFERENCE_TOKENS : (uint32_t) value; - }(); - const uint32_t pipeline_n = force_mm && pin_ref_tokens > 0 ? pin_ref_tokens : (uint32_t) ne11; + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align(ctx, mmp, ne01, pipeline_n, qx_needs_dequant ? f16_type : src0->type, effective_src1_type)); - const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && pipeline_n > 8; + const vk_subbuffer src0_buf = ggml_vk_tensor_subbuffer(ctx, src0, true); + // When scale is absent, bind src0 as a dummy buffer for binding slot 1 + const vk_subbuffer scale_buf = src1 ? ggml_vk_tensor_subbuffer(ctx, src1, true) : src0_buf; + const vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); - vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline(ctx, mmp, ne01, pipeline_n, aligned, qx_needs_dequant ? f16_type : src0->type, effective_src1_type); + vk_op_turbo_wht_push_constants pc = { + (uint32_t)n_groups, + (uint32_t)group_size, + (uint32_t)direction, + src1 ? 1u : 0u, + (uint32_t)head_dim, + (uint32_t)groups_per_head, + 0u, // src_offset — filled by init_pushconst_tensor_offsets + 0u, // dst_offset — filled by init_pushconst_tensor_offsets + 0u, // scale_offset — filled by init_pushconst_tensor_offsets + }; + init_pushconst_tensor_offsets(ctx, pc, src0, src1, nullptr, nullptr, dst); - if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { - pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); - } + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, scale_buf, dst_buf }, pc, { (uint32_t)n_groups, 1, 1 }); +} - // Reserve extra storage in the N dimension for the Y matrix, so we can avoid bounds-checking - uint32_t padded_n = qy_needs_dequant ? ROUNDUP_POW2(ne11, pipeline->wg_denoms[1]) : ne11; - const uint64_t x_ne = ggml_nelements(src0); - // 128 elements per Q8_1 x4 block - const uint64_t y_ne = padded_n * ne10 * ne12 * ne13; - const uint64_t d_ne = ggml_nelements(dst); +// Pinned MUL_MAT through the mat-vec shader in fixed column groups. Each group is a plain +// column window of src1/dst (same buffer, offset data pointer) dispatched as its own +// single-node graph, so ggml_vk_mul_mat_vec_q_f16 sees ne11 <= GGML_VK_MUL_MAT_PIN_GROUP. +static void ggml_vk_mul_mat_pinned_vec(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const int64_t ne11 = src1->ne[1]; + const int64_t group = GGML_VK_MUL_MAT_PIN_GROUP; - const uint32_t split_k = force_mm ? 1 : ggml_vk_guess_split_k(ctx, ne01, ne11, ne10, disable_split_k, pipeline); + const uint8_t * y_base = (const uint8_t *) (src1->view_src ? src1->view_src->data : src1->data) + (src1->view_src ? src1->view_offs : 0); + uint8_t * d_base = (uint8_t *) (dst->view_src ? dst->view_src->data : dst->data) + (dst->view_src ? dst->view_offs : 0); - const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type); - const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type); - const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne; - const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : (y_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne); - const uint64_t d_sz = sizeof(float) * d_ne; + const int saved_fused = ctx->num_additional_fused_ops; + ctx->num_additional_fused_ops = 0; - vk_pipeline to_fp16_vk_0 = nullptr; - vk_pipeline to_fp16_vk_1 = nullptr; - vk_pipeline to_q8_1 = nullptr; + for (int64_t c0 = 0; c0 < ne11; c0 += group) { + const int64_t nc = std::min(group, ne11 - c0); - if (x_non_contig) { - to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0, nullptr, f16_type); - } else { - to_fp16_vk_0 = ggml_vk_get_to_fp16(ctx, src0->type); - } - if (y_non_contig) { - to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, nullptr, f16_type); - } else { - to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); - } - GGML_ASSERT(!qx_needs_dequant || to_fp16_vk_0 != nullptr); // NOLINT - GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT + ggml_tensor y = *src1; + ggml_tensor d = *dst; + y.ne[1] = nc; y.ne[2] = 1; y.ne[3] = 1; + d.ne[1] = nc; d.ne[2] = 1; d.ne[3] = 1; + y.nb[2] = y.nb[1] * nc; y.nb[3] = y.nb[2]; + d.nb[2] = d.nb[1] * nc; d.nb[3] = d.nb[2]; + y.data = (void *) (y_base + c0 * src1->nb[1]); + d.data = (void *) (d_base + c0 * dst->nb[1]); + y.view_src = nullptr; y.view_offs = 0; y.op = GGML_OP_NONE; + d.view_src = nullptr; d.view_offs = 0; + for (auto & sp : y.src) { sp = nullptr; } + for (auto & sp : d.src) { sp = nullptr; } + d.src[0] = src0; + d.src[1] = &y; - if (quantize_y) { - to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); + ggml_tensor * nodes[1] = { &d }; + ggml_cgraph g = {}; + g.n_nodes = 1; + g.nodes = nodes; + + // the prealloc_y reuse cache is keyed on the src1 pointer; the window tensor lives on + // the stack at the same address every iteration, so it must be re-quantized each time + ctx->prealloc_y_last_tensor_used = nullptr; + ggml_vk_mul_mat_vec_q_f16(ctx, subctx, &g, 0, false, true); } - { - const uint64_t split_k_size = split_k > 1 ? d_sz * split_k : 0; - if ( - (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) || - (qy_needs_dequant && y_sz > ctx->device->properties.limits.maxStorageBufferRange) || - (split_k > 1 && split_k_size > ctx->device->properties.limits.maxStorageBufferRange)) { - GGML_ABORT("Requested preallocation size is too large"); - } - if (qx_needs_dequant && ctx->prealloc_size_x < x_sz) { - ctx->prealloc_size_x = x_sz; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if ((qy_needs_dequant || quantize_y) && ctx->prealloc_size_y < y_sz) { - ctx->prealloc_size_y = y_sz; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if (split_k > 1 && ctx->prealloc_size_split_k < split_k_size) { - ctx->prealloc_size_split_k = split_k_size; - ggml_vk_preallocate_buffers(ctx, subctx); - } + ctx->num_additional_fused_ops = saved_fused; +} - // Request descriptor sets - if (qx_needs_dequant) { - ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_0, 1); - } - if (qy_needs_dequant) { - ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_1, 1); - } - if (quantize_y) { - ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1); - } - if (split_k > 1) { - ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, 1); - } - } +void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { + ggml_tensor * dst = cgraph->nodes[node_idx]; + ggml_tensor * src0 = dst->src[0]; + ggml_tensor * src1 = dst->src[1]; + const bool force_mm = ggml_get_op_params_i32(dst, 1) == GGML_HINT_MUL_MAT_PIN; + VK_LOG_DEBUG("ggml_vk_mul_mat(" << src0 << ", " << src1 << ", " << dst << ")"); - ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; - ggml_backend_vk_buffer_context * src0_buf_ctx = (ggml_backend_vk_buffer_context *)src0->buffer->context; - ggml_backend_vk_buffer_context * src1_buf_ctx = (ggml_backend_vk_buffer_context *)src1->buffer->context; + // Handle huge A matrix by splitting the M dimensions. This works well for convolution use cases + // where the M dimension is very large. + // Split_k doesn't work with M splitting. + // This only supports batchsize == 1. + const size_t nbytes = ggml_nbytes(src0); + const bool needs_split = dst->ne[2] == 1 && dst->ne[3] == 1 && nbytes > ctx->device->properties.limits.maxStorageBufferRange; + if (needs_split) { + // Choose the number of rows that can fit (and divide by two, to allow for any additional offsets) + const uint32_t M_split = ctx->device->properties.limits.maxStorageBufferRange / (2 * src0->nb[1]); + uint32_t m_offset = 0; + while (m_offset < dst->ne[0]) { + const uint32_t cur_M_size = std::min(M_split, (uint32_t)(dst->ne[0] - m_offset)); + ggml_tensor dst2 = *dst; + ggml_tensor src02 = *src0; - vk_buffer d_D = dst_buf_ctx->dev_buffer; - const uint64_t d_buf_offset = vk_tensor_offset(dst) + dst->view_offs; - GGML_ASSERT(d_D != nullptr); - GGML_ASSERT(d_D->size >= d_buf_offset + d_sz); - vk_buffer d_Qx = nullptr; - size_t qx_buf_offset = 0; - vk_buffer d_Qy = nullptr; - size_t qy_buf_offset = 0; - bool src0_uma = false; - bool src1_uma = false; + dst2.view_src = dst->view_src ? dst->view_src : dst; + src02.view_src = src0->view_src ? src0->view_src : src0; - if (ctx->device->uma) { - ggml_vk_host_get(ctx->device, src0->data, d_Qx, qx_buf_offset); - ggml_vk_host_get(ctx->device, src1->data, d_Qy, qy_buf_offset); - src0_uma = d_Qx != nullptr; - src1_uma = d_Qy != nullptr; - } - if (!src0_uma) { - d_Qx = src0_buf_ctx->dev_buffer; - qx_buf_offset = vk_tensor_offset(src0) + src0->view_offs; - GGML_ASSERT(d_Qx != nullptr); - } - if (!src1_uma) { - d_Qy = src1_buf_ctx->dev_buffer; - qy_buf_offset = vk_tensor_offset(src1) + src1->view_offs; - GGML_ASSERT(d_Qy != nullptr); - } + dst2.view_offs += m_offset * dst->nb[0]; + src02.view_offs += m_offset * src0->nb[1]; + dst2.ne[0] = cur_M_size; + src02.ne[1] = cur_M_size; - vk_subbuffer d_Qx_copy = ggml_vk_mark_dynamic_subbuffer( - ctx, ggml_vk_subbuffer(ctx, d_Qx, qx_buf_offset), src0); - vk_subbuffer d_Qy_copy = ggml_vk_mark_dynamic_subbuffer( - ctx, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), src1); - vk_subbuffer d_Qx_dequant = ggml_vk_mark_dynamic_subbuffer( - ctx, { d_Qx, qx_buf_offset, qx_sz }, src0); - vk_subbuffer d_Qy_dequant = ggml_vk_mark_dynamic_subbuffer( - ctx, { d_Qy, qy_buf_offset, qy_sz }, src1); - vk_subbuffer d_D_buf = ggml_vk_mark_dynamic_subbuffer( - ctx, ggml_vk_subbuffer(ctx, d_D, d_buf_offset), dst); + ggml_vk_mul_mat_q_f16(ctx, subctx, &src02, src1, &dst2, true, force_mm); - vk_buffer d_X; - uint64_t x_buf_offset = 0; - vk_buffer d_Y; - uint64_t y_buf_offset = 0; - if (qx_needs_dequant) { - d_X = ctx->prealloc_x; - GGML_ASSERT(d_X->size >= x_sz); - } else { - d_X = d_Qx; - x_buf_offset = qx_buf_offset; - GGML_ASSERT(qx_sz == x_sz); - } - if (qy_needs_dequant) { - d_Y = ctx->prealloc_y; - GGML_ASSERT(d_Y->size >= y_sz); - } else if (quantize_y) { - d_Y = ctx->prealloc_y; - GGML_ASSERT(d_Y->size >= CEIL_DIV(y_sz, 144) * 144); + m_offset += cur_M_size; + } + } else if (ggml_vk_can_use_fwht(ctx, src1, dst)) { + ggml_vk_fwht(ctx, subctx, src1, dst); + } else if (force_mm) { + // GGML_HINT_MUL_MAT_PIN: bit-identical output regardless of ne11. Default = the mat-vec + // shader in fixed groups of GGML_VK_MUL_MAT_PIN_GROUP columns; every column's reduction + // is independent of NUM_COLS, so a shorter tail group needs no padding (verified by + // test-wp-mul-mat-pin). GGML_VK_PIN_KERNEL=mm keeps the matrix-pipeline pin for A/B. + static const bool pin_mm = [] { + const char * env = std::getenv("GGML_VK_PIN_KERNEL"); + return env != nullptr && std::strcmp(env, "mm") == 0; + }(); + // plan recording keeps tensor pointers for replay-time buffer resolution; the group + // windows below are stack tensors, so a recorded plan takes the matrix pin instead + const bool mmv_ok = !pin_mm && ctx->recording_plan == nullptr && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || ggml_is_quantized(src0->type)) && + src1->ne[2] * src1->ne[3] == 1 && dst->ne[2] * dst->ne[3] == 1 && + ggml_vk_dim01_contiguous(src1) && ggml_is_contiguous(dst) && + ctx->device->mul_mat_vec_max_cols_eff >= GGML_VK_MUL_MAT_PIN_GROUP; + if (!mmv_ok) { + ggml_vk_mul_mat_q_f16(ctx, subctx, src0, src1, dst, true, true); + } else { + ggml_vk_mul_mat_pinned_vec(ctx, subctx, src0, src1, dst); + } + } else if (src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && dst->ne[1] == 1 && + // detect 0213 permutation, and batch size of 1 + src0->nb[0] <= src0->nb[2] && + src0->nb[2] <= src0->nb[1] && + src0->nb[1] <= src0->nb[3] && + src1->nb[0] <= src1->nb[2] && + src1->nb[2] <= src1->nb[1] && + src1->nb[1] <= src1->nb[3] && + src0->ne[3] == 1 && + src1->ne[3] == 1 && + src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && + src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) { + ggml_vk_mul_mat_vec_p021_f16_f32(ctx, subctx, cgraph, node_idx); + } else if (src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && dst->ne[1] == 1 && + !ggml_is_permuted(src0) && !ggml_is_permuted(src1) && + src0->ne[3] <= ctx->device->properties.limits.maxComputeWorkGroupCount[0] && + src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && + src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) { + ggml_vk_mul_mat_vec_nc_f16_f32(ctx, subctx, cgraph, node_idx); + // With one output row, B^T*A has the same flat output as A^T*B. + } else if (ctx->num_additional_fused_ops == 0 && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && + (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16 || ggml_is_quantized(src1->type)) && + dst->ne[0] == 1 && dst->ne[1] > ctx->device->mul_mat_vec_max_cols_eff && + src0->ne[2] == 1 && src0->ne[3] == 1 && + src1->ne[2] == 1 && src1->ne[3] == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) && + get_misalign_bytes(ctx, src0) == 0 && get_misalign_bytes(ctx, src1) == 0 && get_misalign_bytes(ctx, dst) == 0) { + ggml_vk_mul_mat_vec_q_f16(ctx, subctx, cgraph, node_idx, true); + // mul_mat_vec supports batching ne12*ne13 when ne11==1, or treating ne11 as the batch size (up to four) + // when ne12 and ne13 are one. + } else if ((dst->ne[1] == 1 || (dst->ne[1] <= ctx->device->mul_mat_vec_max_cols_eff && src1->ne[2] * src1->ne[3] == 1)) && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || ggml_is_quantized(src0->type))) { + ggml_vk_mul_mat_vec_q_f16(ctx, subctx, cgraph, node_idx); } else { - d_Y = d_Qy; - y_buf_offset = qy_buf_offset; - GGML_ASSERT(qy_sz == y_sz); + ggml_vk_mul_mat_q_f16(ctx, subctx, src0, src1, dst, false, false); } +} - vk_subbuffer d_X_buf = { d_X, x_buf_offset, x_sz }; - vk_subbuffer d_Y_buf = { d_Y, y_buf_offset, y_sz }; - if (!qx_needs_dequant) { - d_X_buf = ggml_vk_mark_dynamic_subbuffer(ctx, d_X_buf, src0); - } - if (!qy_needs_dequant && !quantize_y) { - d_Y_buf = ggml_vk_mark_dynamic_subbuffer(ctx, d_Y_buf, src1); - } +static constexpr uint32_t GGML_VK_MUL_MAT_ID_FORCE_MM_REFERENCE_TOKENS = GGML_VK_MUL_MAT_FORCE_MM_REFERENCE_TOKENS; - if (x_non_contig || qx_needs_dequant) { - if (ctx->prealloc_x_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); +static bool ggml_vk_mul_mat_id_force_mm(const uint64_t total_tokens) { + static const bool enabled = [] { + const char * env = std::getenv("GGML_MUL_MAT_ID_FORCE_MM"); + return env != nullptr && std::strtol(env, nullptr, 10) != 0; + }(); + static const uint64_t min_tokens = [] { + const char * env = std::getenv("GGML_MUL_MAT_ID_FORCE_MM_MIN_TOKENS"); + if (env == nullptr) { + return uint64_t(64); } - } + char * end = nullptr; + const long value = std::strtol(env, &end, 10); + return end != env && *end == '\0' && value >= 0 ? uint64_t(value) : uint64_t(64); + }(); + return enabled && total_tokens >= min_tokens; +} - if (x_non_contig) { - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, - d_Qx_copy, ggml_vk_subbuffer(ctx, d_X, 0)); - } else if (qx_needs_dequant) { - const std::vector<uint32_t> pc = { (uint32_t)ne01, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)(ggml_nelements(src0)) }; - ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, { d_Qx_dequant, { d_X, 0, x_sz } }, pc, { (uint32_t)(x_ne), 1, 1}); - ggml_vk_sync_buffers(ctx, subctx); - } - if (y_non_contig) { - if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, - d_Qy_copy, ggml_vk_subbuffer(ctx, d_Y, 0)); - ctx->prealloc_y_last_pipeline_used = to_fp16_vk_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; - } - } - if (quantize_y) { - if (ctx->prealloc_y_last_pipeline_used != to_q8_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - ggml_vk_quantize_q8_1(ctx, subctx, d_Qy_copy, - ggml_vk_subbuffer(ctx, d_Y, 0), y_ne); - ctx->prealloc_y_last_pipeline_used = to_q8_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; - } - } +// GGML_HINT_MUL_MAT_PIN on a MUL_MAT_ID op: same pin as GGML_MUL_MAT_ID_FORCE_MM, op-driven +static bool ggml_vk_mul_mat_id_hint_pinned(const ggml_tensor * dst) { + return dst->op == GGML_OP_MUL_MAT_ID && ggml_get_op_params_i32(dst, 1) == GGML_HINT_MUL_MAT_PIN; +} - uint32_t stride_batch_x = ne00*ne01; - uint32_t stride_batch_y = ne10*ne11; +// The MoE expert worker's grouped prefill shape: one activation row per +// (route, token) pair, so src1 is [K, nei0, nei1] rather than the usual +// [K, 1, n_tokens] decode shape. Detected structurally (src1->ne[1] == nei0 > 1 +// and src1->ne[2] == nei1), never assumed from the hint alone. +static bool ggml_vk_mul_mat_id_expanded_rows(const ggml_tensor * dst) { + const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * ids = dst->src[2]; + return ids->ne[0] > 1 && + src1->ne[1] == ids->ne[0] && + src1->ne[2] == ids->ne[1] && + src1->ne[3] == 1; +} - if (!ggml_vk_dim01_contiguous(src0) && !qx_needs_dequant) { - stride_batch_x = src0->nb[0] / ggml_type_size(src0->type); - } +// GGML_VK_MUL_MAT_ID_PIN_KERNEL = mm | vec. Which implementation a *pinned* +// (GGML_HINT_MUL_MAT_PIN) MUL_MAT_ID takes. "mm" (default) keeps the matmul-id +// tile shader; "vec" lets the expanded prefill shape go through +// ggml_vk_mul_mat_vec_id_q_f16 instead. Ignored for unpinned ops. +enum vk_mul_mat_id_pin_kernel { VK_MUL_MAT_ID_PIN_KERNEL_MM, VK_MUL_MAT_ID_PIN_KERNEL_VEC }; - if (!ggml_vk_dim01_contiguous(src1) && !qy_needs_dequant && !quantize_y) { - stride_batch_y = src1->nb[0] / ggml_type_size(src1->type); - } +static vk_mul_mat_id_pin_kernel ggml_vk_mul_mat_id_pin_kernel() { + static const vk_mul_mat_id_pin_kernel kernel = [] { + const char * env = std::getenv("GGML_VK_MUL_MAT_ID_PIN_KERNEL"); + if (env != nullptr && std::strcmp(env, "vec") == 0) { + return VK_MUL_MAT_ID_PIN_KERNEL_VEC; + } + return VK_MUL_MAT_ID_PIN_KERNEL_MM; + }(); + return kernel; +} - // compute - ggml_vk_matmul( - ctx, subctx, pipeline, - std::move(d_X_buf), std::move(d_Y_buf), std::move(d_D_buf), { ctx->prealloc_split_k, 0, d_sz * split_k }, - ne01, ne11, ne10, - ne10, ne10, stride_d, stride_batch_x, stride_batch_y, stride_batch_d, - split_k, ne12*ne13, ne02, ne12, r2, r3, padded_n - ); // NOLINT +// GGML_VK_MUL_MAT_ID_PIN_TOKENS: the N handed to ggml_vk_guess_matmul_id_pipeline +// for a pinned MUL_MAT_ID, i.e. which matmul-id tile gets picked. 0 or unset = +// "auto" (the estimate below). 512 reproduces the pre-change behaviour exactly. +// Only consulted when the pin hint is set. +static uint32_t ggml_vk_mul_mat_id_pin_tokens_override() { + static const uint32_t value = [] { + const char * env = std::getenv("GGML_VK_MUL_MAT_ID_PIN_TOKENS"); + if (env == nullptr) { + return uint32_t(0); + } + char * end = nullptr; + const long parsed = std::strtol(env, &end, 10); + return (end != env && *end == '\0' && parsed > 0) ? (uint32_t) parsed : uint32_t(0); + }(); + return value; +} - if (x_non_contig || qx_needs_dequant) { - ctx->prealloc_x_need_sync = true; +// How many B rows one expert actually gets, which is what the matmul-id tile's N +// dimension should be sized for -- NOT the token count. The pin hint used to +// force N = GGML_VK_MUL_MAT_ID_FORCE_MM_REFERENCE_TOKENS (512), which always +// selects the large 128x128 tile; with ~7 rows per expert that spends 121/128 of +// every tile column on padding. Total selected rows is nei0*nei1, spread over at +// most n_as experts, so nei0*nei1/n_as is a lower bound on the average. It is a +// lower bound (an arena slab has more slots than are routed to), which biases +// towards the smaller tile; sweep GGML_VK_MUL_MAT_ID_PIN_TOKENS to check. +static uint32_t ggml_vk_mul_mat_id_pin_pipeline_n(uint64_t nei0, uint64_t nei1, uint64_t n_as) { + const uint32_t override_n = ggml_vk_mul_mat_id_pin_tokens_override(); + if (override_n != 0) { + return override_n; } - if (y_non_contig || quantize_y) { - ctx->prealloc_y_need_sync = true; + if (n_as == 0) { + return GGML_VK_MUL_MAT_ID_FORCE_MM_REFERENCE_TOKENS; } + const uint64_t rows_per_expert = CEIL_DIV(nei0 * nei1, n_as); + // Floor of 9, not 1: ggml_vk_mul_mat_id_q_f16 only takes the *aligned* + // pipeline variant when pipeline_n > 8, and dropping to the unaligned + // shader is a separate (slower) change that would confound the tile-size + // measurement. 9..32 all select the small 32x32 tile anyway. + return (uint32_t) std::min<uint64_t>(std::max<uint64_t>(rows_per_expert, 9), + GGML_VK_MUL_MAT_ID_FORCE_MM_REFERENCE_TOKENS); } -// Device tuning -static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_t n, uint32_t k, ggml_type src0_type) { - if (device->mmvq_mode == 1) { - return true; - } else if (device->mmvq_mode == -1) { - return false; - } - - // q6_k only has 2-byte alignment which makes it somewhat problematic, - // using MMVQ is only a win on Intel. - bool mmvq_q6 = device->vendor_id == VK_VENDOR_ID_INTEL; - if (src0_type == GGML_TYPE_Q6_K && !mmvq_q6) { - return false; - } - - // MMVQ is generally good for batches - if (n > 1) { - return true; - } - - // Quantization overhead is not worth it for small k - switch (device->vendor_id) { - case VK_VENDOR_ID_NVIDIA: - if (src0_type == GGML_TYPE_Q2_0 || src0_type == GGML_TYPE_Q2_K || src0_type == GGML_TYPE_Q3_K || src0_type == GGML_TYPE_IQ1_S || src0_type == GGML_TYPE_IQ1_M) { - return true; - } - - if (k <= 4096) { - return false; - } - - switch (src0_type) { - case GGML_TYPE_MXFP4: - case GGML_TYPE_Q8_0: - return device->architecture == vk_device_architecture::NVIDIA_PRE_TURING; - default: - return true; - } - case VK_VENDOR_ID_AMD: - if (k < 2048) { - return false; - } - - switch (src0_type) { - case GGML_TYPE_Q8_0: - return device->architecture == vk_device_architecture::AMD_GCN; - default: - return true; - } - case VK_VENDOR_ID_INTEL: - if (device->architecture == vk_device_architecture::INTEL_XE2) { - if (src0_type == GGML_TYPE_Q2_0 || src0_type == GGML_TYPE_Q2_K || src0_type == GGML_TYPE_Q3_K || src0_type == GGML_TYPE_Q6_K) { - return true; - } - } - - if (device->driver_id == vk::DriverId::eIntelProprietaryWindows) { - // Intel Windows proprietary driver MMVQ performance for !Q2/Q3/Q6 is worse than fp16, - // see https://github.com/ggml-org/llama.cpp/issues/17628 and - // https://github.com/ggml-org/llama.cpp/pull/23056 - return false; - } - - if (k < 2048) { - return false; - } - - switch (src0_type) { - // From tests on A770 Linux, may need more tuning - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q5_1: - return false; - default: - return true; - } - default: - return true; - } - - GGML_UNUSED(m); -} - -static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, bool pin = false) { - ggml_tensor * dst = cgraph->nodes[node_idx]; - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - VK_LOG_DEBUG("ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; +static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) { + VK_LOG_DEBUG("ggml_vk_mul_mat_id_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; - std::cerr << ")),)"); - GGML_ASSERT(ggml_vk_dim01_contiguous(src0) || src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); // NOLINT + std::cerr << "), (" << ids << ", name=" << ids->name << ", type=" << ids->type << ", ne0=" << ids->ne[0] << ", ne1=" << ids->ne[1] << ", ne2=" << ids->ne[2] << ", ne3=" << ids->ne[3] << ", nb0=" << ids->nb[0] << ", nb1=" << ids->nb[1] << ", nb2=" << ids->nb[2] << ", nb3=" << ids->nb[3]; + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)"); GGML_ASSERT(ggml_vk_dim01_contiguous(src1) || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); // NOLINT + GGML_ASSERT(ids->type == GGML_TYPE_I32); const uint64_t ne00 = src0->ne[0]; const uint64_t ne01 = src0->ne[1]; const uint64_t ne02 = src0->ne[2]; - const uint64_t ne03 = src0->ne[3]; + // const uint64_t ne03 = src0->ne[3]; const uint64_t ne10 = src1->ne[0]; const uint64_t ne11 = src1->ne[1]; const uint64_t ne12 = src1->ne[2]; const uint64_t ne13 = src1->ne[3]; + const uint64_t nei0 = ids->ne[0]; + const uint64_t nei1 = ids->ne[1]; + const bool pinned = ggml_vk_mul_mat_id_hint_pinned(dst); + const bool force_mm = ggml_vk_mul_mat_id_force_mm(ne12) || pinned; + // A pinned op sizes the tile from the per-expert row count, not from the + // token count: see ggml_vk_mul_mat_id_pin_pipeline_n. GGML_MUL_MAT_ID_FORCE_MM + // (the env-driven pin) keeps its 512-token reference so its behaviour is + // untouched; so does every unpinned op. + const uint32_t pipeline_n = pinned ? ggml_vk_mul_mat_id_pin_pipeline_n(nei0, nei1, ne02) : + force_mm ? GGML_VK_MUL_MAT_ID_FORCE_MM_REFERENCE_TOKENS : (uint32_t) nei1; + + const uint32_t nbi0 = ids->nb[0]; + const uint32_t nbi1 = ids->nb[1]; + const uint32_t nbi2 = ids->nb[2]; + const uint64_t ne20 = dst->ne[0]; const uint64_t ne21 = dst->ne[1]; // const uint64_t ne22 = dst->ne[2]; // const uint64_t ne23 = dst->ne[3]; - const uint64_t r2 = ne12 / ne02; - const uint64_t r3 = ne13 / ne03; + const uint64_t n_as = ne02; + // n_as counts, n_as offsets, one total, then one packed row id per (expert, token). + // Hoisting requires 16-bit indices for the packing and a table that fits one binding. + const uint64_t hoisted_row_id_words = 2 * n_as + 1 + nei0 * nei1; + // 1024 matches MAX_EXPERTS in count_experts.comp and LLAMA_MAX_EXPERTS. It costs + // 3 * 1024 * 4 = 12 KiB of shared memory, within the 16 KiB Vulkan guarantees. + const bool hoist_row_ids = n_as <= 1024 && nei0 <= 0xffff && nei1 <= 0xffff && + hoisted_row_id_words * sizeof(uint32_t) <= + ctx->device->properties.limits.maxStorageBufferRange; - // batch_n indicates that we need to compute a few vector results, and this assumes - // ne12 and ne13 are 1. It overloads the batch_strides to hold the row strides. - GGML_ASSERT(ne11 == 1 || ne12 * ne13 == 1); - bool batch_n = ne11 > 1; + ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; + ggml_backend_vk_buffer_context * src0_buf_ctx = (ggml_backend_vk_buffer_context *)src0->buffer->context; + ggml_backend_vk_buffer_context * src1_buf_ctx = (ggml_backend_vk_buffer_context *)src1->buffer->context; + ggml_backend_vk_buffer_context * ids_buf_ctx = (ggml_backend_vk_buffer_context *)ids->buffer->context; - const bool x_non_contig = !ggml_vk_dim01_contiguous(src0); - const bool y_non_contig = !ggml_vk_dim01_contiguous(src1); + vk_buffer d_Qx = nullptr; + size_t qx_buf_offset = 0; + vk_buffer d_Qy = nullptr; + size_t qy_buf_offset = 0; + vk_buffer d_ids = nullptr; + size_t ids_buf_offset = 0; - const bool f16_f32_kernel = src1->type == GGML_TYPE_F32; - // pinned (GGML_HINT_MUL_MAT_PIN) groups evaluate the mmvq decision at the fixed group width, so the - // y-quantization choice cannot change with the real column count - const uint64_t ne11_decide = pin ? GGML_VK_MUL_MAT_PIN_GROUP : ne11; - bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11_decide * ne10) % 4 == 0 && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11_decide, ne10, src0->type); + bool src0_uma = false; + bool src1_uma = false; + bool ids_uma = false; - vk_pipeline to_fp16_vk_0 = nullptr; - vk_pipeline to_fp16_vk_1 = nullptr; - if (x_non_contig) { - to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0, nullptr, src0->type); - } - if (y_non_contig) { - to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, nullptr, src1->type); - } else { - to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); + if (ctx->device->uma) { + ggml_vk_host_get(ctx->device, src0->data, d_Qx, qx_buf_offset); + ggml_vk_host_get(ctx->device, src1->data, d_Qy, qy_buf_offset); + ggml_vk_host_get(ctx->device, ids->data, d_ids, ids_buf_offset); + src0_uma = d_Qx != nullptr; + src1_uma = d_Qy != nullptr; + ids_uma = d_ids != nullptr; } + // Reformat and convert to fp16 if non-contiguous, or for coopmat2 for better perf + const bool x_non_contig = (ctx->device->coopmat2 && src0->type == GGML_TYPE_F32) || + !ggml_vk_dim01_contiguous(src0); + // If src0 is BF16, try to use a BF16 x BF16 multiply + ggml_type f16_type = src0->type == GGML_TYPE_BF16 ? GGML_TYPE_BF16 : GGML_TYPE_F16; +#if defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT) + // B must already be, or be convertible to, the matmul B type used by this path. + const bool y_decode_vector_supported = ctx->device->coopmat2_decode_vector && + (f16_type != GGML_TYPE_BF16 || ctx->device->coopmat2_bf16_support) && + (src1->type == GGML_TYPE_F32 || src1->type == f16_type); + // If B is copied to prealloc_y, we can choose a 4-element-aligned row stride. + const bool y_decode_vector_uses_prealloc = !ggml_vk_dim01_contiguous(src1) || src1->type != f16_type; + // Direct B reads are safe only if row starts and the original buffer offset are 4-element aligned. + const bool y_decode_vector_aligned = + (ne10 % 4 == 0) && + (y_decode_vector_uses_prealloc || get_misalign_bytes(ctx, src1) % (4 * ggml_type_size(src1->type)) == 0); + // Stage B only when decode-vector is available and direct B reads would be misaligned. + const bool y_decode_vector_staging = y_decode_vector_supported && !y_decode_vector_aligned; +#else + const bool y_decode_vector_staging = false; +#endif + const bool y_non_contig = y_decode_vector_staging || + (ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) || + // Intel coopmat1: force f32->f16 conversion so the f16 B-type quant pipeline is used. + (ctx->device->coopmat_support && !ctx->device->coopmat2 && + ctx->device->vendor_id == VK_VENDOR_ID_INTEL && + ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) || + (src0->type == GGML_TYPE_BF16 && src1->type != GGML_TYPE_BF16) || + !ggml_vk_dim01_contiguous(src1); + + const bool y_f32_kernel = src1->type == GGML_TYPE_F32 && !y_non_contig; + + bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0; + // Check for mmq first - vk_pipeline dmmv = quantize_y ? ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, GGML_TYPE_Q8_1, ne11, ne20, ne00) : nullptr; - vk_pipeline to_q8_1 = nullptr; + const std::vector<vk_matmul_pipeline_pair>* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0], true) : nullptr; - if (dmmv == nullptr) { + if (mmp_map == nullptr) { // Fall back to f16 dequant mul mat - dmmv = ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, src1->type, ne11, ne20, ne00); + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0], true); quantize_y = false; } - if (quantize_y) { - to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); - } + const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig; + bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig); - if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { - dmmv = ggml_vk_get_64b_indexing_pipeline(ctx, dmmv); + if (qx_needs_dequant) { + // Fall back to dequant + f16 mulmat + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0], true); } - const bool qx_needs_dequant = x_non_contig; - const bool qy_needs_dequant = !quantize_y && ((src1->type != GGML_TYPE_F16 && !f16_f32_kernel) || y_non_contig); + // Coopmat2 MUL_MAT_ID BK specialization constants in ggml_vk_load_shaders are at most 64. + const uint32_t y_staged_row_stride = ctx->device->coopmat2 && !quantize_y ? ggml_vk_align_size(ne10, 64) : ne10; + const bool y_needs_k_padding = ne10 != y_staged_row_stride; + const bool y_needs_reformat = y_non_contig || y_needs_k_padding; + qy_needs_dequant = qy_needs_dequant || y_needs_k_padding; // Not implemented - GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT + GGML_ASSERT(y_needs_reformat || !qy_needs_dequant); // NOLINT - GGML_ASSERT(!qx_needs_dequant || to_fp16_vk_0 != nullptr); // NOLINT - GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT - GGML_ASSERT(dmmv != nullptr); + GGML_ASSERT(mmp_map != nullptr); - const uint64_t x_ne = ggml_nelements(src0); - const uint64_t y_ne = ggml_nelements(src1); + // note: pipeline_n is nei1 unless the pin path forces a reference token count + // (ggml_vk_mul_mat_id_pin_pipeline_n), which keeps the tile choice stable. + const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, pipeline_n, true)); + const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && pipeline_n > 8; - const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device->properties.limits.minStorageBufferOffsetAlignment); - const uint64_t x_sz = x_non_contig ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : qx_sz; - const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : - (f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne); + vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, pipeline_n, aligned, true); - { - if ( - (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) || - (qy_needs_dequant && y_sz > ctx->device->properties.limits.maxStorageBufferRange)) { - GGML_ABORT("Requested preallocation size is too large"); - } - if (qx_needs_dequant && ctx->prealloc_size_x < x_sz) { - ctx->prealloc_size_x = x_sz; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if ((qy_needs_dequant || quantize_y) && ctx->prealloc_size_y < y_sz) { - ctx->prealloc_size_y = y_sz; - ggml_vk_preallocate_buffers(ctx, subctx); - } + if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { + pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); + } - // Request descriptor sets - if (qx_needs_dequant) { - ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_0, 1); - } - if (qy_needs_dequant) { - ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_1, 1); - } - if (quantize_y) { - ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1); + // GGML_VK_MUL_MAT_ID_PIN_TRACE=1: which tile a pinned mul_mat_id actually got. + static const bool pin_trace = std::getenv("GGML_VK_MUL_MAT_ID_PIN_TRACE") != nullptr; + if (pin_trace && pinned) { + static int n_traced = 0; + if (n_traced < 8) { + ++n_traced; + fprintf(stderr, "[vk-mmid-pin] mm pipeline=%s wg_denoms={%u,%u} pipeline_n=%u aligned=%d " + "m=%u nei0=%u nei1=%u n_as=%u\n", + pipeline->name.c_str(), pipeline->wg_denoms[0], pipeline->wg_denoms[1], + pipeline_n, (int) aligned, (uint32_t) ne01, + (uint32_t) nei0, (uint32_t) nei1, (uint32_t) ne02); } } - vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, cgraph->nodes[node_idx + ctx->num_additional_fused_ops]); - vk_subbuffer d_Qx = ggml_vk_tensor_subbuffer(ctx, src0); - vk_subbuffer d_Qy = ggml_vk_tensor_subbuffer(ctx, src1); - vk_subbuffer d_X, d_Y; + const uint64_t x_ne = ggml_nelements(src0); + const uint64_t y_ne = (uint64_t)y_staged_row_stride * ne11 * ne12 * ne13; + const uint64_t d_ne = ggml_nelements(dst); + + const uint64_t src0_type_size = ggml_type_size(src0->type); + const uint64_t src0_blck_size = ggml_blck_size(src0->type); + const uint64_t src0_canonical_nb2 = + (ne00 * ne01 / src0_blck_size) * src0_type_size; + const bool src0_strided = src0->nb[2] != src0_canonical_nb2; + const bool src0_stride_representable = src0_type_size != 0 && + src0_blck_size != 0 && src0->nb[2] % src0_type_size == 0 && + (src0->nb[2] / src0_type_size) <= UINT32_MAX / src0_blck_size; + const bool src0_arena_stride = src0_strided && src0_stride_representable; + // The A-dequant staging step below (qx_needs_dequant) flattens src0 with + // ggml_nelements()/x_ne and writes a DENSE fp16 copy: it has no notion of + // nb[2], so for an arena slab it would decode the wrong bytes for every + // slot past the first, and size prealloc_x by the whole slab. Fail loudly + // instead. This cannot fire while a native mul_mat_id pipeline exists for + // (src0->type, f32/f16), which is the case for every expert weight type. + GGML_ASSERT((!src0_arena_stride || !qx_needs_dequant) && + "arena-strided src0 requires a native mul_mat_id pipeline"); + const uint64_t qx_sz = src0_arena_stride ? ggml_nbytes(src0) : + src0_type_size * x_ne / src0_blck_size; + const uint64_t qy_sz = ggml_type_size(src1->type) * ggml_nelements(src1) / ggml_blck_size(src1->type); + const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne; + const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : (y_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne); + const uint64_t ids_sz = nbi2; + const uint64_t d_sz = sizeof(float) * d_ne; + + vk_pipeline to_fp16_vk_0 = nullptr; + vk_pipeline to_fp16_vk_1 = nullptr; + vk_pipeline to_q8_1 = nullptr; + + auto make_y_staged_dst = [&]() { + ggml_tensor y_staged_dst = *src1; + y_staged_dst.type = f16_type; + y_staged_dst.nb[0] = ggml_type_size(f16_type); + y_staged_dst.nb[1] = y_staged_dst.nb[0] * y_staged_row_stride; + y_staged_dst.nb[2] = y_staged_dst.nb[1] * ne11; + y_staged_dst.nb[3] = y_staged_dst.nb[2] * y_staged_dst.ne[2]; + return y_staged_dst; + }; + + if (x_non_contig) { + to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0, nullptr, f16_type); + } else { + to_fp16_vk_0 = ggml_vk_get_to_fp16(ctx, src0->type); + } + if (y_needs_reformat) { + ggml_tensor y_staged_dst; + const ggml_tensor * y_staged_dst_ptr = nullptr; + if (y_needs_k_padding) { + y_staged_dst = make_y_staged_dst(); + y_staged_dst_ptr = &y_staged_dst; + } + + to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, y_staged_dst_ptr, f16_type); + } else { + to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); + } + GGML_ASSERT(!qx_needs_dequant || to_fp16_vk_0 != nullptr); // NOLINT + GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT + if (quantize_y) { + to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); + } + vk_pipeline count_experts = ctx->device->pipeline_count_experts; + + const size_t expert_data_size = sizeof(uint32_t) * + (hoist_row_ids ? hoisted_row_id_words : n_as); + + { + if ( + (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) || + (qy_needs_dequant && y_sz > ctx->device->properties.limits.maxStorageBufferRange)) { + GGML_ABORT("Requested preallocation size is too large"); + } + if (qx_needs_dequant && ctx->prealloc_size_x < x_sz) { + ctx->prealloc_size_x = x_sz; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if ((qy_needs_dequant || quantize_y) && ctx->prealloc_size_y < y_sz) { + ctx->prealloc_size_y = y_sz; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if (ctx->prealloc_size_split_k < expert_data_size) { + ctx->prealloc_size_split_k = expert_data_size; + ggml_vk_preallocate_buffers(ctx, subctx); + } + + // Request descriptor sets + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + if (qx_needs_dequant) { + ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_0, 1); + } + if (qy_needs_dequant) { + ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_1, 1); + } + if (quantize_y) { + ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1); + } + ggml_pipeline_request_descriptor_sets(ctx, count_experts, 1); + } + + vk_buffer d_D = dst_buf_ctx->dev_buffer; + const uint64_t d_buf_offset = vk_tensor_offset(dst) + dst->view_offs; + GGML_ASSERT(d_D != nullptr); + vk_buffer d_X; + uint64_t x_buf_offset = 0; + vk_buffer d_Y; + uint64_t y_buf_offset = 0; + if (!src0_uma) { + d_Qx = src0_buf_ctx->dev_buffer; + qx_buf_offset = vk_tensor_offset(src0) + src0->view_offs; + GGML_ASSERT(d_Qx != nullptr); + } + if (!src1_uma) { + d_Qy = src1_buf_ctx->dev_buffer; + qy_buf_offset = vk_tensor_offset(src1) + src1->view_offs; + GGML_ASSERT(d_Qy != nullptr); + } + if (!ids_uma) { + d_ids = ids_buf_ctx->dev_buffer; + ids_buf_offset = vk_tensor_offset(ids) + ids->view_offs; + GGML_ASSERT(d_ids != nullptr); + } if (qx_needs_dequant) { - d_X = { ctx->prealloc_x, 0, ctx->prealloc_x->size }; + d_X = ctx->prealloc_x; + GGML_ASSERT(d_X->size >= x_sz); } else { d_X = d_Qx; + x_buf_offset = qx_buf_offset; GGML_ASSERT(qx_sz == x_sz); } - if (qy_needs_dequant || quantize_y) { - d_Y = { ctx->prealloc_y, 0, ctx->prealloc_y->size }; + if (qy_needs_dequant) { + d_Y = ctx->prealloc_y; + GGML_ASSERT(d_Y->size >= y_sz); + } else if (quantize_y) { + d_Y = ctx->prealloc_y; + GGML_ASSERT(d_Y->size >= CEIL_DIV(y_sz, 144) * 144); } else { d_Y = d_Qy; + y_buf_offset = qy_buf_offset; + GGML_ASSERT(qy_sz == y_sz); } - if (x_non_contig) { + if (x_non_contig || qx_needs_dequant) { if (ctx->prealloc_x_need_sync) { ggml_vk_sync_buffers(ctx, subctx); } + } + // Count how many times each expert is used + vk_subbuffer expert_count_buf = { ctx->prealloc_split_k, 0, expert_data_size }; + if (ctx->prealloc_split_k_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + { + vk_op_count_experts_push_constants pc = { (uint32_t)nei0, + (uint32_t)nei1, + (uint32_t)(nbi0 / ggml_type_size(ids->type)), + (uint32_t)(nbi1 / ggml_type_size(ids->type)), + (uint32_t)(get_misalign_bytes(ctx, ids) / ggml_type_size(ids->type)), + (uint32_t)n_as, + uint32_t(hoist_row_ids), + 0, 0 }; + init_pushconst_fastdiv(pc); + ggml_vk_dispatch_pipeline(ctx, subctx, count_experts, + { vk_subbuffer{ d_ids, ids_buf_offset, ids_sz }, expert_count_buf }, pc, + { hoist_row_ids ? 1u : (uint32_t)n_as, 1, 1}); + } - GGML_ASSERT(x_sz == ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment)); - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, d_Qx, d_X); + if (x_non_contig) { + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, ggml_vk_subbuffer(ctx, d_Qx, qx_buf_offset), ggml_vk_subbuffer(ctx, d_X, 0)); + } else if (qx_needs_dequant) { + const std::vector<uint32_t> pc = { (uint32_t)ne01, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)(ggml_nelements(src0)) }; + ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, + { vk_subbuffer{ d_Qx, qx_buf_offset, qx_sz }, vk_subbuffer{ d_X, 0, x_sz } }, pc, { (uint32_t)x_ne, 1, 1}); } - if (y_non_contig) { - GGML_ASSERT(y_sz == ggml_type_size(src1->type) * y_ne); + if (y_needs_reformat) { if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { + ctx->prealloc_y_last_k_padded != y_needs_k_padding) { if (ctx->prealloc_y_need_sync) { ggml_vk_sync_buffers(ctx, subctx); } - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, d_Qy, d_Y); + if (y_needs_k_padding) { + GGML_ASSERT(y_sz % 4 == 0); + // Zero B padding because clamping only A can produce 0 * Inf or NaN. + subctx->s->buffer->buf.fillBuffer(d_Y->buffer, 0, y_sz, 0); + ggml_vk_sync_buffers(ctx, subctx); + const ggml_tensor y_staged_dst = make_y_staged_dst(); + const uint32_t y_staged_dst_type_size = ggml_type_size(y_staged_dst.type); + ggml_vk_cpy_to_strided( + ctx, subctx, to_fp16_vk_1, src1, + ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), ggml_vk_subbuffer(ctx, d_Y, 0), + (uint32_t)(y_staged_dst.nb[0] / y_staged_dst_type_size), + (uint32_t)(y_staged_dst.nb[1] / y_staged_dst_type_size), + (uint32_t)(y_staged_dst.nb[2] / y_staged_dst_type_size), + (uint32_t)(y_staged_dst.nb[3] / y_staged_dst_type_size)); + } else { + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), ggml_vk_subbuffer(ctx, d_Y, 0)); + } ctx->prealloc_y_last_pipeline_used = to_fp16_vk_1.get(); ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; + ctx->prealloc_y_last_k_padded = y_needs_k_padding; } } if (quantize_y) { @@ -11560,6628 +8945,5024 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& if (ctx->prealloc_y_need_sync) { ggml_vk_sync_buffers(ctx, subctx); } - ggml_vk_quantize_q8_1(ctx, subctx, d_Qy, d_Y, y_ne); + ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), ggml_vk_subbuffer(ctx, d_Y, 0), y_ne); ctx->prealloc_y_last_pipeline_used = to_q8_1.get(); ctx->prealloc_y_last_tensor_used = src1; ctx->prealloc_y_last_k_padded = false; } } + ggml_vk_sync_buffers(ctx, subctx); - // For batch_n, the A matrix is the same for each batch, and B/D use the row stride as the batch stride - uint32_t stride_batch_x = batch_n ? 0 : ne00*ne01; - uint32_t stride_batch_y = batch_n ? ne10 : (ne10*ne11); - uint32_t stride_batch_d = batch_n ? ne20 : (ne20*ne21); + uint32_t stride_batch_x = ne00*ne01; + if (src0_arena_stride) { + const uint64_t stride_batch_x64 = + (src0->nb[2] / src0_type_size) * src0_blck_size; + GGML_ASSERT(src0_type_size != 0 && src0->nb[2] % src0_type_size == 0); // NOLINT + GGML_ASSERT(stride_batch_x64 <= UINT32_MAX); // NOLINT + stride_batch_x = (uint32_t) stride_batch_x64; + } + uint32_t stride_b_y = y_needs_k_padding ? y_staged_row_stride : ne10; + uint32_t stride_batch_y = y_needs_k_padding ? y_staged_row_stride * ne11 : ne10*ne11; if (!ggml_vk_dim01_contiguous(src0) && !qx_needs_dequant) { stride_batch_x = src0->nb[0] / ggml_type_size(src0->type); } - if (!ggml_vk_dim01_contiguous(src1) && !qy_needs_dequant) { + if (!ggml_vk_dim01_contiguous(src1) && !qy_needs_dequant && !quantize_y) { stride_batch_y = src1->nb[0] / ggml_type_size(src1->type); } - const uint32_t max_groups_x = ctx->device->properties.limits.maxComputeWorkGroupCount[0]; - - uint32_t groups_x = ne01; - uint32_t groups_z = 1; - - if (ne01 > max_groups_x) { - groups_z = 64; - groups_x = CEIL_DIV(groups_x, groups_z); + // Weight paging, prefill path. Same contract as the mul_mat_vec_id path, but + // this shader indexes data_a in units of LOAD_VEC_A elements rather than in + // quant blocks, so republish the offsets in ELEMENTS and let the shader + // divide by LOAD_VEC_A_EFF exactly as it does for batch_stride_a. The block + // count times the block's element count is an exact element offset. + vk_wp_expert_offsets wp_off; + bool wp_paged = ggml_vk_wp_take_expert_offsets(wp_off); + if (wp_paged && qx_needs_dequant) { + // The dequant staging step reads src0 at its single patched slot, so the + // paged offsets cannot be honoured. Fall through to stock addressing: + // wrong for a paged model, but wrong visibly rather than silently. + GGML_LOG_WARN("%s: weight paging: mul_mat_id needs A dequant for %s, paged offsets ignored\n", + __func__, ggml_get_name(src0)); + wp_paged = false; + } + vk_subbuffer d_WP = { d_ids, ids_buf_offset, ids_sz }; // dummy when not paged + vk_buffer x_buf = d_X; + uint64_t x_off = x_buf_offset; + uint64_t x_len = x_sz; + if (wp_paged) { + const uint32_t elems_per_block = (uint32_t) ggml_blck_size(src0->type); + std::vector<uint32_t> elem_off(wp_off.offsets.size()); + for (size_t e = 0; e < elem_off.size(); ++e) { + elem_off[e] = wp_off.offsets[e] * elems_per_block; + } + d_WP = ggml_vk_wp_upload_expert_offsets(ctx, elem_off); + if (wp_off.pool_buffer != nullptr) { + auto * pool_ctx = (ggml_backend_vk_buffer_context *) wp_off.pool_buffer->context; + x_buf = pool_ctx->dev_buffer; + x_off = 0; + x_len = pool_ctx->dev_buffer->size; + } } - uint32_t fusion_flags = 0; - - vk_subbuffer d_F0 = d_D; - if (ctx->num_additional_fused_ops > 0) { - const ggml_tensor * add = cgraph->nodes[node_idx + 1]; - const ggml_tensor * bias = add->src[0] == dst ? add->src[1] : add->src[0]; - - d_F0 = ggml_vk_tensor_subbuffer(ctx, bias); - fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; - } - - vk_subbuffer d_F1 = d_D; - if (ctx->num_additional_fused_ops == 2) { - const ggml_tensor * add = cgraph->nodes[node_idx + 2]; - const ggml_tensor * bias = add->src[0] == cgraph->nodes[node_idx + 1] ? add->src[1] : add->src[0]; - - d_F1 = ggml_vk_tensor_subbuffer(ctx, bias); - fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS1; - } - - ggml_pipeline_request_descriptor_sets(ctx, dmmv, CEIL_DIV(ne12 * ne13, ctx->device->properties.limits.maxComputeWorkGroupCount[1])); - - uint32_t base_work_group_y = 0; - while (base_work_group_y < ne12 * ne13) { - - uint32_t groups_y = std::min((uint32_t)(ne12 * ne13) - base_work_group_y, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); - const vk_mat_vec_push_constants pc = { - (uint32_t)ne00, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne01, - stride_batch_x, stride_batch_y, stride_batch_d, - fusion_flags, base_work_group_y, - (uint32_t)ne02, (uint32_t)ne12, (uint32_t)r2, (uint32_t)r3, - }; - ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, - { - d_X, - d_Y, - d_D, - d_F0, - d_F1, - }, - pc, { groups_x, groups_y, groups_z }); - base_work_group_y += groups_y; - } + // compute + ggml_vk_matmul_id( + ctx, subctx, pipeline, + { x_buf, x_off, x_len }, { d_Y, y_buf_offset, y_sz }, + { d_D, d_buf_offset, d_sz }, { d_ids, ids_buf_offset, ids_sz }, expert_count_buf, + d_WP, + ne01, ne21, ne10, ne10, stride_b_y, ne01, + stride_batch_x, stride_batch_y, ne20*ne21, + n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, hoist_row_ids, + wp_paged ? 1u : 0u + ); // NOLINT - if (x_non_contig) { + if (x_non_contig || qx_needs_dequant) { ctx->prealloc_x_need_sync = true; } - if (y_non_contig || quantize_y) { + if (y_needs_reformat || quantize_y) { ctx->prealloc_y_need_sync = true; } + ctx->prealloc_split_k_need_sync = true; } -static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { +static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { ggml_tensor * dst = cgraph->nodes[node_idx]; - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - VK_LOG_DEBUG("ggml_vk_mul_mat_p021_f16_f32(" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; + ggml_tensor * src0 = dst->src[0]; + ggml_tensor * src1 = dst->src[1]; + ggml_tensor * ids = dst->src[2]; + VK_LOG_DEBUG("ggml_vk_mul_mat_vec_id_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; + std::cerr << "), (" << ids << ", name=" << ids->name << ", type=" << ids->type << ", ne0=" << ids->ne[0] << ", ne1=" << ids->ne[1] << ", ne2=" << ids->ne[2] << ", ne3=" << ids->ne[3] << ", nb0=" << ids->nb[0] << ", nb1=" << ids->nb[1] << ", nb2=" << ids->nb[2] << ", nb3=" << ids->nb[3]; std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; std::cerr << "))"); - GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1)); - GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // NOLINT - GGML_ASSERT(src1->nb[0] <= src1->nb[1] && src1->nb[2] <= src1->nb[3]); // NOLINT - GGML_ASSERT(src0->type == GGML_TYPE_F16); - GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_vk_dim01_contiguous(src0) || src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); // NOLINT + GGML_ASSERT(ggml_vk_dim01_contiguous(src1) || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); // NOLINT + GGML_ASSERT(ids->type == GGML_TYPE_I32); const uint64_t ne00 = src0->ne[0]; const uint64_t ne01 = src0->ne[1]; - const uint64_t ne02 = src0->ne[2]; + // const uint64_t ne02 = src0->ne[2]; // const uint64_t ne03 = src0->ne[3]; - //const uint64_t ne10 = src1->ne[0]; + const uint64_t src0_type_size = ggml_type_size(src0->type); + const uint64_t src0_blck_size = ggml_blck_size(src0->type); + const uint64_t src0_canonical_nb2 = + (ne00 * ne01 / src0_blck_size) * src0_type_size; + const bool src0_strided = src0->nb[2] != src0_canonical_nb2; + const bool src0_stride_representable = src0_type_size != 0 && + src0_blck_size != 0 && src0->nb[2] % src0_type_size == 0 && + (src0->nb[2] / src0_type_size) <= UINT32_MAX / src0_blck_size; + const bool src0_arena_stride = src0_strided && src0_stride_representable; + uint32_t stride_batch_x = (uint32_t)(ne00 * ne01); + if (src0_arena_stride) { + const uint64_t stride_batch_x64 = + (src0->nb[2] / src0_type_size) * src0_blck_size; + GGML_ASSERT(stride_batch_x64 <= UINT32_MAX); // NOLINT + stride_batch_x = (uint32_t) stride_batch_x64; + } + + const uint64_t ne10 = src1->ne[0]; const uint64_t ne11 = src1->ne[1]; const uint64_t ne12 = src1->ne[2]; // const uint64_t ne13 = src1->ne[3]; - GGML_ASSERT(ne11 == 1); + const uint64_t nei0 = ids->ne[0]; + const uint64_t nei1 = ids->ne[1]; + const uint32_t nbi1 = (uint32_t)(ids->nb[1] / sizeof(int)); - // With grouped query attention there are > 1 Q matrices per K, V matrix. - uint32_t gqa_ratio = (uint32_t)ne12 / (uint32_t)ne02; - if (gqa_ratio > 8 || gqa_ratio == 0 || ne12 != ne02 * gqa_ratio) { - gqa_ratio = 1; + const uint64_t ne20 = dst->ne[0]; + const uint64_t ne21 = dst->ne[1]; + // const uint64_t ne22 = dst->ne[2]; + // const uint64_t ne23 = dst->ne[3]; + + const bool x_non_contig = !ggml_vk_dim01_contiguous(src0); + const bool y_non_contig = !ggml_vk_dim01_contiguous(src1); + + const bool f16_f32_kernel = src1->type == GGML_TYPE_F32; + bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type); + + vk_pipeline to_fp16_vk_0 = nullptr; + vk_pipeline to_fp16_vk_1 = nullptr; + if (x_non_contig) { + to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0, nullptr, src0->type); + } + if (y_non_contig) { + to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, nullptr, src1->type); + } else { + to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); } - vk_pipeline pipeline = ctx->device->pipeline_mul_mat_vec_p021_f16_f32[gqa_ratio - 1]; + // Check for mmq first + vk_pipeline dmmv = quantize_y ? ggml_vk_get_dequantize_mul_mat_vec_id(ctx, src0->type, GGML_TYPE_Q8_1, ne20, ne00) : nullptr; + vk_pipeline to_q8_1 = nullptr; + + if (dmmv == nullptr) { + // Fall back to f16 dequant mul mat + dmmv = ggml_vk_get_dequantize_mul_mat_vec_id(ctx, src0->type, src1->type, ne20, ne00); + quantize_y = false; + } + + if (quantize_y) { + to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); + } + + const bool qx_needs_dequant = x_non_contig; + const bool qy_needs_dequant = !quantize_y && ((src1->type != GGML_TYPE_F16 && !f16_f32_kernel) || y_non_contig); if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { - pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); + dmmv = ggml_vk_get_64b_indexing_pipeline(ctx, dmmv); } + // Not implemented + GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT + GGML_ASSERT(!qx_needs_dequant || to_fp16_vk_0 != nullptr); // NOLINT + GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT + GGML_ASSERT(dmmv != nullptr); + + const uint64_t x_ne = ggml_nelements(src0); + const uint64_t y_ne = ggml_nelements(src1); + + const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device->properties.limits.minStorageBufferOffsetAlignment); + const uint64_t x_sz = x_non_contig ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : qx_sz; + const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : + (f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne); + { + if ( + (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) || + (qy_needs_dequant && y_sz > ctx->device->properties.limits.maxStorageBufferRange)) { + GGML_ABORT("Requested preallocation size is too large"); + } + if (qx_needs_dequant && ctx->prealloc_size_x < x_sz) { + ctx->prealloc_size_x = x_sz; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if ((qy_needs_dequant || quantize_y) && ctx->prealloc_size_y < y_sz) { + ctx->prealloc_size_y = y_sz; + ggml_vk_preallocate_buffers(ctx, subctx); + } + // Request descriptor sets - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + if (qx_needs_dequant) { + ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_0, 1); + } + if (qy_needs_dequant) { + ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_1, 1); + } + if (quantize_y) { + ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1); + } + ggml_pipeline_request_descriptor_sets(ctx, dmmv, nei1); } - vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, cgraph->nodes[node_idx + ctx->num_additional_fused_ops], true); + vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, cgraph->nodes[node_idx + ctx->num_additional_fused_ops]); vk_subbuffer d_Qx = ggml_vk_tensor_subbuffer(ctx, src0); - vk_subbuffer d_Qy = ggml_vk_tensor_subbuffer(ctx, src1, true); - + vk_subbuffer d_Qy = ggml_vk_tensor_subbuffer(ctx, src1); + vk_subbuffer d_ids = ggml_vk_tensor_subbuffer(ctx, ids); vk_subbuffer d_F0 = d_D; + vk_subbuffer d_X, d_Y; - uint32_t fusion_flags = 0; + if (qx_needs_dequant) { + d_X = { ctx->prealloc_x, 0, ctx->prealloc_x->size }; + } else { + d_X = d_Qx; + } + if (qy_needs_dequant || quantize_y) { + d_Y = { ctx->prealloc_y, 0, ctx->prealloc_y->size }; + } else { + d_Y = d_Qy; + } - if (ctx->num_additional_fused_ops > 0) { - const ggml_tensor * add = cgraph->nodes[node_idx + 1]; - const ggml_tensor * bias = add->src[0] == dst ? add->src[1] : add->src[0]; + if (x_non_contig) { + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + } - d_F0 = ggml_vk_tensor_subbuffer(ctx, bias); - fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; + if (x_non_contig) { + GGML_ASSERT(x_sz == ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment)); + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, d_Qx, d_X); + } + if (y_non_contig) { + GGML_ASSERT(y_sz == ggml_type_size(src1->type) * y_ne); + if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || + ctx->prealloc_y_last_tensor_used != src1 || + ctx->prealloc_y_last_k_padded) { + if (ctx->prealloc_y_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, d_Qy, d_Y); + ctx->prealloc_y_last_pipeline_used = to_fp16_vk_1.get(); + ctx->prealloc_y_last_tensor_used = src1; + ctx->prealloc_y_last_k_padded = false; + } + } + if (quantize_y) { + if (ctx->prealloc_y_last_pipeline_used != to_q8_1.get() || + ctx->prealloc_y_last_tensor_used != src1 || + ctx->prealloc_y_last_k_padded) { + if (ctx->prealloc_y_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + ggml_vk_quantize_q8_1(ctx, subctx, d_Qy, d_Y, y_ne); + ctx->prealloc_y_last_pipeline_used = to_q8_1.get(); + ctx->prealloc_y_last_tensor_used = src1; + ctx->prealloc_y_last_k_padded = false; + } } - vk_subbuffer d_F1 = d_D; - if (ctx->num_additional_fused_ops > 1) { - const ggml_tensor * bias = cgraph->nodes[node_idx + 2]->src[1]; + uint32_t stride_batch_y = ne10*ne11; - d_F1 = ggml_vk_tensor_subbuffer(ctx, bias); - fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS1; + if (!ggml_vk_dim01_contiguous(src1) && !qy_needs_dequant) { + stride_batch_y = src1->nb[2] / ggml_type_size(src1->type); } - // compute - - vk_mat_vec_p021_push_constants pc = { - (uint32_t)ne00, (uint32_t)ne01, (uint32_t)ne02, (uint32_t)ne12, - 0, 0, fusion_flags - }; + const uint32_t max_groups_x = ctx->device->properties.limits.maxComputeWorkGroupCount[0]; - init_pushconst_tensor_offsets(ctx, pc, src0, src1, nullptr, nullptr, cgraph->nodes[node_idx + ctx->num_additional_fused_ops]); - - uint32_t workgroups_z = (uint32_t)ne12; - // When gqa_ratio > 1, each invocation does multiple rows and we can launch fewer workgroups - if (gqa_ratio > 1) { - workgroups_z /= gqa_ratio; - } - - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - { - d_Qx, - d_Qy, - d_D, - d_F0, - d_F1, - }, pc, { 1, (uint32_t)ne01, workgroups_z }); -} - -static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { - ggml_tensor * dst = cgraph->nodes[node_idx]; - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - VK_LOG_DEBUG("ggml_vk_mul_mat_nc_f16_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; - std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; - std::cerr << "))"); - GGML_ASSERT(!ggml_is_transposed(src0)); - GGML_ASSERT(!ggml_is_transposed(src1)); - GGML_ASSERT(!ggml_is_permuted(src0)); - GGML_ASSERT(src0->type == GGML_TYPE_F16); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - - const uint64_t ne00 = src0->ne[0]; - const uint64_t ne01 = src0->ne[1]; - const uint64_t ne02 = src0->ne[2]; - const uint64_t ne03 = src0->ne[3]; - - const uint64_t nb01 = src0->nb[1]; - const uint64_t nb02 = src0->nb[2]; - - const uint64_t nb12 = src1->nb[2]; - - // const uint64_t ne10 = src1->ne[0]; - const uint64_t ne11 = src1->ne[1]; - const uint64_t ne12 = src1->ne[2]; - // const uint64_t ne13 = src1->ne[3]; - - const uint32_t nb03 = (uint32_t)(src0->nb[3] / sizeof(ggml_fp16_t)); - const uint32_t nb13 = (uint32_t)(src1->nb[3] / sizeof(float)); - const uint32_t nb23 = (uint32_t)(dst->nb[3] / sizeof(float)); - - GGML_ASSERT(ne11 == 1); - GGML_ASSERT(src0->ne[3] == src1->ne[3]); // checked in supports_op - - const uint32_t row_stride_x = nb01 / sizeof(ggml_fp16_t); - const uint32_t channel_stride_x = nb02 / sizeof(ggml_fp16_t); - const uint32_t channel_stride_y = nb12 / sizeof(float); - - vk_pipeline pipeline = ctx->device->pipeline_mul_mat_vec_nc_f16_f32; - if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { - pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); - } + uint32_t groups_x = ne01; + uint32_t groups_z = 1; - { - // Request descriptor sets - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + if (ne01 > max_groups_x) { + groups_z = 64; + groups_x = CEIL_DIV(groups_x, groups_z); } - vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, cgraph->nodes[node_idx + ctx->num_additional_fused_ops], true); - vk_subbuffer d_Qx = ggml_vk_tensor_subbuffer(ctx, src0); - vk_subbuffer d_Qy = ggml_vk_tensor_subbuffer(ctx, src1, true); - vk_subbuffer d_F0 = d_D; - uint32_t fusion_flags = 0; if (ctx->num_additional_fused_ops > 0) { - const ggml_tensor * add = cgraph->nodes[node_idx + 1]; - const ggml_tensor * bias = add->src[0] == dst ? add->src[1] : add->src[0]; + const ggml_tensor * bias = cgraph->nodes[node_idx + 1]->src[1]; d_F0 = ggml_vk_tensor_subbuffer(ctx, bias); - fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; + + if (cgraph->nodes[node_idx + 1]->op == GGML_OP_MUL) { + fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE0; + } else { + GGML_ASSERT(cgraph->nodes[node_idx + 1]->op == GGML_OP_ADD_ID); + fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; + } } vk_subbuffer d_F1 = d_D; if (ctx->num_additional_fused_ops > 1) { - const ggml_tensor * bias = cgraph->nodes[node_idx + 2]->src[1]; - - d_F1 = ggml_vk_tensor_subbuffer(ctx, bias); - fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS1; - } - - // compute - vk_mat_vec_nc_push_constants pc = { - (uint32_t)ne00, (uint32_t)ne01, - row_stride_x, channel_stride_x, channel_stride_y, - (uint32_t)(ne12 / ne02), (uint32_t)ne12, - 0, 0, - nb03, nb13, nb23, fusion_flags - }; - - init_pushconst_tensor_offsets(ctx, pc, src0, src1, nullptr, nullptr, cgraph->nodes[node_idx + ctx->num_additional_fused_ops]); - - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - { - d_Qx, - d_Qy, - d_D, - d_F0, - d_F1, - }, pc, { (uint32_t)ne03, (uint32_t)ne01, (uint32_t)ne12 }); -} + const ggml_tensor * scale = cgraph->nodes[node_idx + 2]->src[1]; -static int ggml_vk_fwht_pipeline_idx(int64_t n) { - switch (n) { - case 64: return 0; - case 128: return 1; - case 256: return 2; - case 512: return 3; - default: return -1; + d_F1 = ggml_vk_tensor_subbuffer(ctx, scale); + fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE1; } -} -static bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor * src1, const ggml_tensor * dst) { - if (ctx->num_additional_fused_ops != 0) { - return false; - } + // Weight paging: if the pager published offsets for this node, the experts + // live at arbitrary slots in its pool rather than at a uniform stride in + // src0, so bind the pool and hand the shader the per-slot offsets. Only + // applies when the active-expert count fits the push constants; otherwise + // fall through to stock addressing, which for a paged model is wrong but + // visibly so rather than silently. + vk_wp_expert_offsets wp_off; + bool wp_paged = ggml_vk_wp_take_expert_offsets(wp_off); - if (ggml_get_op_params_i32(dst, 1) != GGML_HINT_SRC0_IS_HADAMARD) { - return false; + // GGML_VK_WP_COVERAGE=1: is EVERY expert matmul actually getting a + // publication? An expert node with no publication silently falls through to + // stock uniform-stride addressing, which is wrong for a paged model because + // src0->data has been repointed at a single pool slot. The existing TRACE + // only prints the first 8 dispatches and cannot see this. + static const bool wp_coverage = getenv("GGML_VK_WP_COVERAGE") != nullptr; + if (wp_coverage) { + static size_t n_total = 0, n_paged = 0, n_unpaged_exps = 0; + ++n_total; + if (wp_paged) { + ++n_paged; + } else { + const char * nm = ggml_get_name(src0); + if (nm != nullptr && strstr(nm, "exps") != nullptr) { + if (n_unpaged_exps < 8) { + fprintf(stderr, "[vk-wp-coverage] UNPAGED expert matmul src0=%s\n", nm); + } + ++n_unpaged_exps; + } + } + if (n_total % 100 == 0) { + fprintf(stderr, "[vk-wp-coverage] mul_mat_id dispatches=%zu paged=%zu " + "unpaged_exps=%zu\n", n_total, n_paged, n_unpaged_exps); + } } - - const int idx = ggml_vk_fwht_pipeline_idx(src1->ne[0]); - if (idx < 0 || ctx->device->pipeline_fwht_f32[idx] == nullptr) { - return false; + // GGML_VK_WP_FORCE=1: exercise the paged shader path on a NON-paged model, + // publishing offsets that reproduce the stock stride and leaving data_a on + // src0. Correct output here proves the flag/SSBO/shader plumbing; garbage + // proves the plumbing itself is broken, independent of the pager. + static const bool wp_trace = getenv("GGML_VK_WP_TRACE") != nullptr; + if (wp_trace) { + static int n = 0; + if (n < 8) { + ++n; + fprintf(stderr, "[vk-wp-trace] dispatch src0=%-28s paged=%d off0=%u\n", + ggml_get_name(src0), (int) wp_paged, + wp_off.offsets.empty() ? 0u : wp_off.offsets[0]); + } } - - if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return false; + static const bool wp_force = getenv("GGML_VK_WP_FORCE") != nullptr; + if (wp_force && !wp_paged) { + const uint32_t blocks_per_expert = + (uint32_t) ((ne00 * ne01) / ggml_blck_size(src0->type)); + wp_off.offsets.resize((size_t) src0->ne[2]); + for (size_t e = 0; e < wp_off.offsets.size(); ++e) { + wp_off.offsets[e] = (uint32_t) e * blocks_per_expert; + } + wp_off.pool_buffer = nullptr; + wp_paged = true; } - - if (!ggml_is_contiguous(src1)) { - return false; + vk_subbuffer d_WP = d_ids; // dummy binding when not paged + // GGML_VK_WP_STOCKOFF=1: keep the paged plumbing (flag + SSBO) but publish + // offsets that reproduce the stock uniform stride and leave data_a on src0. + // Isolates "is the SSBO/flag/shader path correct" from "are the pool + // offsets/binding correct" -- the two halves fail identically otherwise. + static const bool wp_stockoff = getenv("GGML_VK_WP_STOCKOFF") != nullptr; + if (wp_paged && wp_stockoff) { + const uint32_t blocks_per_expert = + (uint32_t) ((ne00 * ne01) / ggml_blck_size(src0->type)); + for (size_t e = 0; e < wp_off.offsets.size(); ++e) { + wp_off.offsets[e] = (uint32_t) e * blocks_per_expert; + } } - GGML_ASSERT(ggml_is_contiguous(dst)); - - return true; -} - -static void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst) { - const int idx = ggml_vk_fwht_pipeline_idx(src->ne[0]); - vk_pipeline pipeline = ctx->device->pipeline_fwht_f32[idx]; - - const uint32_t rows_per_workgroup = 4; - const uint32_t n_rows = (uint32_t)ggml_nrows(src); - const uint32_t max_workgroups_x = ctx->device->properties.limits.maxComputeWorkGroupCount[0]; + // Deliberately NOT provided: knobs that perturb the published offsets to see + // whether output changes (collapse-to-one-slot, rotate-by-one). They read as + // sensitivity tests but are near no-ops on the paged path, because inactive + // experts already share the first_active_slot fallback and the ACTIVE experts + // keep their own correct offsets. Both produce "no change", which invites the + // false conclusion that the shader is ignoring the pool. Use WP_IDCHK and + // WP_EXPDUMP instead: they test expert IDENTITY, which is what offset-keyed + // checks are blind to. + if (wp_paged && !wp_stockoff && wp_off.pool_buffer != nullptr) { + auto * pool_ctx = (ggml_backend_vk_buffer_context *) wp_off.pool_buffer->context; + d_X = vk_subbuffer{ pool_ctx->dev_buffer, 0, pool_ctx->dev_buffer->size }; - const uint32_t total_workgroups = CEIL_DIV(n_rows, rows_per_workgroup); - const uint32_t workgroups_x = std::min(total_workgroups, max_workgroups_x); - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + // GGML_VK_WP_BIND=1: dump what is actually bound as data_a. The shader + // honours p.paged and data_wp_expert_off (proved by FORCE), the offsets + // are exact and the bytes at them are correct, so the remaining question + // is whether binding 0 is really the pool at dispatch time. + static const bool wp_bind = getenv("GGML_VK_WP_BIND") != nullptr; + if (wp_bind) { + static int n = 0; + if (n < 6) { + ++n; + uint32_t first = 0; + for (uint32_t o : wp_off.offsets) { if (o != 0) { first = o; break; } } + const size_t blk = ggml_type_size(src0->type); + fprintf(stderr, + "[vk-wp-bind] src0=%-26s type=%s blk=%zu\n" + " pool_vkbuf=%p pool_size=%zu\n" + " d_X.vkbuf=%p d_X.off=%zu d_X.size=%zu\n" + " src0_vkbuf=%p src0_off=%zu first_off_blocks=%u byte=%zu\n", + ggml_get_name(src0), ggml_type_name(src0->type), blk, + (void *) (VkBuffer) pool_ctx->dev_buffer->buffer, + (size_t) pool_ctx->dev_buffer->size, + (void *) (VkBuffer) d_X.buffer->buffer, + (size_t) d_X.offset, (size_t) d_X.size, + (void *) (VkBuffer) d_Qx.buffer->buffer, (size_t) d_Qx.offset, + first, (size_t) first * blk); + } + } - const vk_subbuffer src_buf = ggml_vk_tensor_subbuffer(ctx, src, true); - const vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); - - vk_op_fwht_push_constants pc = { - n_rows, - 0, - 0, - 1.0f / std::sqrt((float)src->ne[0]), - }; - init_pushconst_tensor_offsets(ctx, pc, src, nullptr, nullptr, nullptr, dst); - - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src_buf, dst_buf }, pc, { workgroups_x, 1, 1 }); -} - -static void ggml_vk_turbo_wht(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; // InnerQ scale_inv (may be nullptr) - - int direction, group_size; - memcpy(&direction, dst->op_params + 0, sizeof(int)); - memcpy(&group_size, dst->op_params + sizeof(int), sizeof(int)); - - const int64_t head_dim = src0->ne[0]; - const int64_t n_heads = ggml_nelements(src0) / head_dim; - const int64_t groups_per_head = head_dim / group_size; - const int64_t n_groups = groups_per_head * n_heads; - - if (n_groups == 0) { - return; + // GGML_VK_WP_HASHCHK=1: re-read the pool at the offsets we are about to + // hand the shader and compare with what stage_in put there. Proves or + // disproves slot-lifetime corruption at the point of use. + if (wp_hashchk_enabled()) { + static size_t n_checked = 0, n_bad = 0, n_untracked = 0, n_dispatch = 0; + const size_t blk = ggml_type_size(src0->type); + if (blk != 0 && ++n_dispatch <= 400) { + std::vector<uint8_t> back(WP_HASH_SPAN); + std::set<uint32_t> seen; + for (uint32_t off_blocks : wp_off.offsets) { + if (!seen.insert(off_blocks).second) { continue; } + const size_t byte_off = (size_t) off_blocks * blk; + if (byte_off + WP_HASH_SPAN > pool_ctx->dev_buffer->size) { continue; } + uint64_t want = 0; + { + std::lock_guard<std::mutex> hg(g_wp_hash_mutex); + auto it = g_wp_slot_hash.find(byte_off); + if (it == g_wp_slot_hash.end()) { ++n_untracked; continue; } + want = it->second; + } + ggml_vk_buffer_read(pool_ctx->dev_buffer, byte_off, back.data(), WP_HASH_SPAN); + const uint64_t got = wp_hash_bytes(back.data(), WP_HASH_SPAN); + ++n_checked; + if (got != want) { + if (n_bad < 8) { + fprintf(stderr, "[vk-wp-hashchk] MISMATCH at pool byte offset %zu " + "(blocks %u, blk %zu) src0=%s\n", + byte_off, off_blocks, blk, ggml_get_name(src0)); + } + ++n_bad; + } + } + if (n_dispatch % 100 == 0) { + fprintf(stderr, "[vk-wp-hashchk] dispatches=%zu offsets_checked=%zu " + "mismatched=%zu untracked=%zu\n", + n_dispatch, n_checked, n_bad, n_untracked); + } + } + } + } + if (wp_paged) { + d_WP = ggml_vk_wp_upload_expert_offsets(ctx, wp_off.offsets); } - vk_pipeline pipeline = ctx->device->pipeline_turbo_wht_f32; - - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - - const vk_subbuffer src0_buf = ggml_vk_tensor_subbuffer(ctx, src0, true); - // When scale is absent, bind src0 as a dummy buffer for binding slot 1 - const vk_subbuffer scale_buf = src1 ? ggml_vk_tensor_subbuffer(ctx, src1, true) : src0_buf; - const vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); - - vk_op_turbo_wht_push_constants pc = { - (uint32_t)n_groups, - (uint32_t)group_size, - (uint32_t)direction, - src1 ? 1u : 0u, - (uint32_t)head_dim, - (uint32_t)groups_per_head, - 0u, // src_offset — filled by init_pushconst_tensor_offsets - 0u, // dst_offset — filled by init_pushconst_tensor_offsets - 0u, // scale_offset — filled by init_pushconst_tensor_offsets - }; - init_pushconst_tensor_offsets(ctx, pc, src0, src1, nullptr, nullptr, dst); - - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, scale_buf, dst_buf }, pc, { (uint32_t)n_groups, 1, 1 }); -} - -// Pinned MUL_MAT through the mat-vec shader in fixed column groups. Each group is a plain -// column window of src1/dst (same buffer, offset data pointer) dispatched as its own -// single-node graph, so ggml_vk_mul_mat_vec_q_f16 sees ne11 <= GGML_VK_MUL_MAT_PIN_GROUP. -static void ggml_vk_mul_mat_pinned_vec(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const int64_t ne11 = src1->ne[1]; - const int64_t group = GGML_VK_MUL_MAT_PIN_GROUP; - - const uint8_t * y_base = (const uint8_t *) (src1->view_src ? src1->view_src->data : src1->data) + (src1->view_src ? src1->view_offs : 0); - uint8_t * d_base = (uint8_t *) (dst->view_src ? dst->view_src->data : dst->data) + (dst->view_src ? dst->view_offs : 0); - - const int saved_fused = ctx->num_additional_fused_ops; - ctx->num_additional_fused_ops = 0; - - for (int64_t c0 = 0; c0 < ne11; c0 += group) { - const int64_t nc = std::min(group, ne11 - c0); - - ggml_tensor y = *src1; - ggml_tensor d = *dst; - y.ne[1] = nc; y.ne[2] = 1; y.ne[3] = 1; - d.ne[1] = nc; d.ne[2] = 1; d.ne[3] = 1; - y.nb[2] = y.nb[1] * nc; y.nb[3] = y.nb[2]; - d.nb[2] = d.nb[1] * nc; d.nb[3] = d.nb[2]; - y.data = (void *) (y_base + c0 * src1->nb[1]); - d.data = (void *) (d_base + c0 * dst->nb[1]); - y.view_src = nullptr; y.view_offs = 0; y.op = GGML_OP_NONE; - d.view_src = nullptr; d.view_offs = 0; - for (auto & sp : y.src) { sp = nullptr; } - for (auto & sp : d.src) { sp = nullptr; } - d.src[0] = src0; - d.src[1] = &y; + // GGML_VK_WP_IDCHK=1: the general identity invariant. Two DISTINCT experts + // selected by the ids tensor can never legitimately live at the same pool + // offset. If they do, at least one of them was never staged and is silently + // reading the first_active_slot fallback -- which is exactly what a + // stale/mismatched routing readback in the eval callback would produce, and + // which every offset-keyed check is blind to. + static const bool wp_idchk = getenv("GGML_VK_WP_IDCHK") != nullptr; + if (wp_paged && wp_idchk) { + static size_t n_disp = 0, n_viol = 0, n_dup_pairs = 0; + ++n_disp; + const size_t n_ids = (size_t) nei0 * (size_t) nei1; + if (n_ids > 0 && n_ids < 4096) { + std::vector<int32_t> ids_host(n_ids); + ggml_vk_buffer_read(d_ids.buffer, d_ids.offset, + ids_host.data(), n_ids * sizeof(int32_t)); + std::map<uint32_t, int32_t> off_to_id; + bool viol = false; + for (int32_t id : ids_host) { + if (id < 0 || (size_t) id >= wp_off.offsets.size()) { continue; } + const uint32_t off = wp_off.offsets[(size_t) id]; + auto it = off_to_id.find(off); + if (it != off_to_id.end() && it->second != id) { + ++n_dup_pairs; + if (!viol && n_viol < 8) { + fprintf(stderr, "[vk-wp-idchk] COLLISION src0=%s: experts %d and %d " + "both map to pool offset %u blocks\n", + ggml_get_name(src0), it->second, id, off); + } + viol = true; + } else { + off_to_id[off] = id; + } + } + if (viol) { ++n_viol; } + } + if (n_disp % 100 == 0) { + fprintf(stderr, "[vk-wp-idchk] dispatches=%zu with_collision=%zu dup_pairs=%zu\n", + n_disp, n_viol, n_dup_pairs); + } + } - ggml_tensor * nodes[1] = { &d }; - ggml_cgraph g = {}; - g.n_nodes = 1; - g.nodes = nodes; + // GGML_VK_WP_EXPDUMP=<path>: dump the first 32 bytes the shader would read + // for EVERY expert of one named tensor, using exactly the addressing the + // dispatch is about to use. Run once paged and once not, then diff: identical + // means expert e really is at the slot published for expert e; a permutation + // means the expert->slot association is wrong, which every other check so far + // is blind to because they are all keyed by offset rather than identity. + { + static const char * dump_path = getenv("GGML_VK_WP_EXPDUMP"); + static bool dumped = false; + const char * nm = ggml_get_name(src0); + if (dump_path != nullptr && !dumped && nm != nullptr && + strcmp(nm, "blk.2.ffn_gate_exps.weight") == 0) { + dumped = true; + const size_t blk = ggml_type_size(src0->type); + const uint32_t bpe = (uint32_t) ((ne00 * ne01) / ggml_blck_size(src0->type)); + FILE * f = fopen(dump_path, "w"); + if (f != nullptr) { + fprintf(f, "# tensor=%s type=%s paged=%d blk=%zu blocks_per_expert=%u\n", + nm, ggml_type_name(src0->type), (int) wp_paged, blk, bpe); + const uint32_t n_exp = (uint32_t) src0->ne[2]; + std::vector<uint8_t> b(32); + for (uint32_t e = 0; e < n_exp; ++e) { + const uint32_t off_blocks = (wp_paged && e < wp_off.offsets.size()) + ? wp_off.offsets[e] : e * bpe; + const size_t abs = (size_t) d_X.offset + (size_t) off_blocks * blk; + if (abs + b.size() > d_X.buffer->size) { continue; } + ggml_vk_buffer_read(d_X.buffer, abs, b.data(), b.size()); + fprintf(f, "expert %2u off_blocks %10u ", e, off_blocks); + for (uint8_t v : b) { fprintf(f, "%02x", v); } + fprintf(f, "\n"); + } + // The experts the SHADER will actually use, straight out of the + // ids tensor at dispatch time. If this set differs from the set + // the pager staged, every selected-but-unstaged expert silently + // reads the first_active_slot fallback. + const size_t n_ids = (size_t) nei0 * (size_t) nei1; + std::vector<int32_t> ids_host(n_ids); + ggml_vk_buffer_read(d_ids.buffer, d_ids.offset, + ids_host.data(), n_ids * sizeof(int32_t)); + fprintf(f, "# ids tensor (nei0=%u nei1=%u):", (uint32_t) nei0, (uint32_t) nei1); + for (int32_t v : ids_host) { fprintf(f, " %d", v); } + fprintf(f, "\n"); + fclose(f); + fprintf(stderr, "[vk-wp-expdump] wrote %s (paged=%d)\n", dump_path, (int) wp_paged); + } + } + } - // the prealloc_y reuse cache is keyed on the src1 pointer; the window tensor lives on - // the stack at the same address every iteration, so it must be re-quantized each time - ctx->prealloc_y_last_tensor_used = nullptr; - ggml_vk_mul_mat_vec_q_f16(ctx, subctx, &g, 0, true); + // Loop over the batch dimension + for (uint32_t expert_i1 = 0; expert_i1 < nei1; ++expert_i1) { + const vk_mat_vec_id_push_constants pc = { + (uint32_t)ne00, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne01, + stride_batch_x, stride_batch_y, (uint32_t)(ne20 * ne21), + fusion_flags, + (uint32_t)nei0, (uint32_t)ne11, expert_i1, nbi1, + wp_paged ? 1u : 0u + }; + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, + { + d_X, + d_Y, + d_D, + d_F0, + d_F1, + d_ids, + d_WP, + }, + pc, { groups_x, (uint32_t)nei0, groups_z }); } - ctx->num_additional_fused_ops = saved_fused; + if (x_non_contig) { + ctx->prealloc_x_need_sync = true; + } + if (y_non_contig || quantize_y) { + ctx->prealloc_y_need_sync = true; + } } -static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { +bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx) { ggml_tensor * dst = cgraph->nodes[node_idx]; ggml_tensor * src0 = dst->src[0]; ggml_tensor * src1 = dst->src[1]; - const bool force_mm = ggml_get_op_params_i32(dst, 1) == GGML_HINT_MUL_MAT_PIN; - VK_LOG_DEBUG("ggml_vk_mul_mat(" << src0 << ", " << src1 << ", " << dst << ")"); - - // Handle huge A matrix by splitting the M dimensions. This works well for convolution use cases - // where the M dimension is very large. - // Split_k doesn't work with M splitting. - // This only supports batchsize == 1. - const size_t nbytes = ggml_nbytes(src0); - const bool needs_split = dst->ne[2] == 1 && dst->ne[3] == 1 && nbytes > ctx->device->properties.limits.maxStorageBufferRange; - if (needs_split) { - // Choose the number of rows that can fit (and divide by two, to allow for any additional offsets) - const uint32_t M_split = ctx->device->properties.limits.maxStorageBufferRange / (2 * src0->nb[1]); - uint32_t m_offset = 0; - while (m_offset < dst->ne[0]) { - const uint32_t cur_M_size = std::min(M_split, (uint32_t)(dst->ne[0] - m_offset)); - ggml_tensor dst2 = *dst; - ggml_tensor src02 = *src0; - - dst2.view_src = dst->view_src ? dst->view_src : dst; - src02.view_src = src0->view_src ? src0->view_src : src0; - - dst2.view_offs += m_offset * dst->nb[0]; - src02.view_offs += m_offset * src0->nb[1]; - dst2.ne[0] = cur_M_size; - src02.ne[1] = cur_M_size; - - ggml_vk_mul_mat_q_f16(ctx, subctx, &src02, src1, &dst2, true, force_mm); + ggml_tensor * src2 = dst->src[2]; + if (ggml_vk_mul_mat_id_force_mm(src1->ne[2])) { + return false; + } + // A pinned op normally takes the matmul-id path (that is what the hint was + // introduced for). GGML_VK_MUL_MAT_ID_PIN_KERNEL=vec opts the *expanded + // prefill* shape into the mat-vec-id path instead: that shader already + // addresses src1 as (route, token) -- b_offset = (expert_i0 % ne11)*stride_b + // + expert_i1*batch_stride_b in mul_mat_vec_base.glsl -- and already honours + // an arena-strided src0 through stride_batch_x, so no copy and no shader + // change is needed. The <= 8 token cap below is a batch-size heuristic for + // the unpinned decode path and does not apply here. + bool pinned_vec = false; + if (ggml_vk_mul_mat_id_hint_pinned(dst)) { + if (ggml_vk_mul_mat_id_pin_kernel() != VK_MUL_MAT_ID_PIN_KERNEL_VEC || + !ggml_vk_mul_mat_id_expanded_rows(dst)) { + return false; + } + pinned_vec = true; + } + const uint64_t type_size = ggml_type_size(src0->type); + const uint64_t block_size = ggml_blck_size(src0->type); + const uint64_t canonical_nb2 = + (src0->ne[0] * src0->ne[1] / block_size) * type_size; + const bool strided = src0->nb[2] != canonical_nb2; + const bool stride_representable = type_size != 0 && block_size != 0 && + src0->nb[2] % type_size == 0 && + (src0->nb[2] / type_size) <= UINT32_MAX / block_size; + const bool supported_type = + src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type); + return supported_type && (pinned_vec || src2->ne[1] <= 8) && + (!strided || stride_representable); +} - m_offset += cur_M_size; +void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { + ggml_tensor * dst = cgraph->nodes[node_idx]; + ggml_tensor * src0 = dst->src[0]; + ggml_tensor * src1 = dst->src[1]; + ggml_tensor * src2 = dst->src[2]; + VK_LOG_DEBUG("ggml_vk_mul_mat_id(" << src0 << ", " << src1 << ", " << src2 << ", " << dst << ")"); + const bool use_vec = ggml_vk_use_mul_mat_vec_id(cgraph, node_idx); + // GGML_VK_WP_FORK=1: which mul_mat_id implementation is actually running. + // The vec path serves batches of <= 8 tokens (plus a pinned expanded-prefill + // op under GGML_VK_MUL_MAT_ID_PIN_KERNEL=vec). MM handles wider batches and + // strided arena weights whose stride cannot be represented by batch_stride_a. + static const bool wp_fork = getenv("GGML_VK_WP_FORK") != nullptr; + if (wp_fork) { + static size_t n_vec = 0, n_mm = 0; + (use_vec ? n_vec : n_mm)++; + if (!use_vec && n_mm <= 4) { + fprintf(stderr, "[vk-wp-fork] mm path src0=%s tokens=%d\n", + ggml_get_name(src0), (int) src2->ne[1]); } - } else if (ggml_vk_can_use_fwht(ctx, src1, dst)) { - ggml_vk_fwht(ctx, subctx, src1, dst); - } else if (force_mm) { - // GGML_HINT_MUL_MAT_PIN: bit-identical output regardless of ne11. Default = the mat-vec - // shader in fixed groups of GGML_VK_MUL_MAT_PIN_GROUP columns; every column's reduction - // is independent of NUM_COLS, so a shorter tail group needs no padding (verified by - // test-wp-mul-mat-pin). GGML_VK_PIN_KERNEL=mm keeps the matrix-pipeline pin for A/B. - static const bool pin_mm = [] { - const char * env = std::getenv("GGML_VK_PIN_KERNEL"); - return env != nullptr && std::strcmp(env, "mm") == 0; - }(); - // plan recording keeps tensor pointers for replay-time buffer resolution; the group - // windows below are stack tensors, so a recorded plan takes the matrix pin instead - const bool mmv_ok = !pin_mm && ctx->recording_plan == nullptr && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || ggml_is_quantized(src0->type)) && - src1->ne[2] * src1->ne[3] == 1 && dst->ne[2] * dst->ne[3] == 1 && - ggml_vk_dim01_contiguous(src1) && ggml_is_contiguous(dst) && - ctx->device->mul_mat_vec_max_cols_eff >= GGML_VK_MUL_MAT_PIN_GROUP; - if (!mmv_ok) { - ggml_vk_mul_mat_q_f16(ctx, subctx, src0, src1, dst, true, true); - } else { - ggml_vk_mul_mat_pinned_vec(ctx, subctx, src0, src1, dst); + if ((n_vec + n_mm) % 100 == 0) { + fprintf(stderr, "[vk-wp-fork] mul_mat_id vec=%zu mm=%zu\n", n_vec, n_mm); } - } else if (src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && dst->ne[1] == 1 && - // detect 0213 permutation, and batch size of 1 - src0->nb[0] <= src0->nb[2] && - src0->nb[2] <= src0->nb[1] && - src0->nb[1] <= src0->nb[3] && - src1->nb[0] <= src1->nb[2] && - src1->nb[2] <= src1->nb[1] && - src1->nb[1] <= src1->nb[3] && - src0->ne[3] == 1 && - src1->ne[3] == 1 && - src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && - src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) { - ggml_vk_mul_mat_vec_p021_f16_f32(ctx, subctx, cgraph, node_idx); - } else if (src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && dst->ne[1] == 1 && - !ggml_is_permuted(src0) && !ggml_is_permuted(src1) && - src0->ne[3] <= ctx->device->properties.limits.maxComputeWorkGroupCount[0] && - src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && - src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) { - ggml_vk_mul_mat_vec_nc_f16_f32(ctx, subctx, cgraph, node_idx); - // mul_mat_vec supports batching ne12*ne13 when ne11==1, or treating ne11 as the batch size (up to four) - // when ne12 and ne13 are one. - } else if ((dst->ne[1] == 1 || (dst->ne[1] <= ctx->device->mul_mat_vec_max_cols_eff && src1->ne[2] * src1->ne[3] == 1)) && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || ggml_is_quantized(src0->type))) { - ggml_vk_mul_mat_vec_q_f16(ctx, subctx, cgraph, node_idx); + } + if (use_vec) { + ggml_vk_mul_mat_vec_id_q_f16(ctx, subctx, cgraph, node_idx); } else { - ggml_vk_mul_mat_q_f16(ctx, subctx, src0, src1, dst, false, false); + ggml_vk_mul_mat_id_q_f16(ctx, subctx, src0, src1, src2, dst); } } -static constexpr uint32_t GGML_VK_MUL_MAT_ID_FORCE_MM_REFERENCE_TOKENS = GGML_VK_MUL_MAT_FORCE_MM_REFERENCE_TOKENS; +bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) { + GGML_UNUSED(f32acc); + // Needs to be kept up to date on shader changes + const uint32_t wg_size = params.workgroup_size; + const uint32_t Br = params.block_rows; + const uint32_t Bc = params.block_cols; -static bool ggml_vk_mul_mat_id_force_mm(const uint64_t total_tokens) { - static const bool enabled = [] { - const char * env = std::getenv("GGML_MUL_MAT_ID_FORCE_MM"); - return env != nullptr && std::strtol(env, nullptr, 10) != 0; - }(); - static const uint64_t min_tokens = [] { - const char * env = std::getenv("GGML_MUL_MAT_ID_FORCE_MM_MIN_TOKENS"); - if (env == nullptr) { - return uint64_t(64); - } - char * end = nullptr; - const long value = std::strtol(env, &end, 10); - return end != env && *end == '\0' && value >= 0 ? uint64_t(value) : uint64_t(64); - }(); - return enabled && total_tokens >= min_tokens; -} + // BF16 uses the fp32 shader (FLOAT_TYPE=float) + const uint32_t float_type_size = (device->fp16 && k_type != GGML_TYPE_BF16) ? sizeof(ggml_fp16_t) : sizeof(float); -// GGML_HINT_MUL_MAT_PIN on a MUL_MAT_ID op: same pin as GGML_MUL_MAT_ID_FORCE_MM, op-driven -static bool ggml_vk_mul_mat_id_hint_pinned(const ggml_tensor * dst) { - return dst->op == GGML_OP_MUL_MAT_ID && ggml_get_op_params_i32(dst, 1) == GGML_HINT_MUL_MAT_PIN; -} + const bool mmq = ggml_vk_fa_scalar_uses_mmq(device, k_type, v_type); -// The MoE expert worker's grouped prefill shape: one activation row per -// (route, token) pair, so src1 is [K, nei0, nei1] rather than the usual -// [K, 1, n_tokens] decode shape. Detected structurally (src1->ne[1] == nei0 > 1 -// and src1->ne[2] == nei1), never assumed from the hint alone. -static bool ggml_vk_mul_mat_id_expanded_rows(const ggml_tensor * dst) { - const ggml_tensor * src1 = dst->src[1]; - const ggml_tensor * ids = dst->src[2]; - return ids->ne[0] > 1 && - src1->ne[1] == ids->ne[0] && - src1->ne[2] == ids->ne[1] && - src1->ne[3] == 1; -} + // tmpsh is overestimated slightly + const uint32_t tmpsh = wg_size * sizeof(float); + const uint32_t tmpshv4 = wg_size * 4 * float_type_size; -// GGML_VK_MUL_MAT_ID_PIN_KERNEL = mm | vec. Which implementation a *pinned* -// (GGML_HINT_MUL_MAT_PIN) MUL_MAT_ID takes. "mm" (default) keeps the matmul-id -// tile shader; "vec" lets the expanded prefill shape go through -// ggml_vk_mul_mat_vec_id_q_f16 instead. Ignored for unpinned ops. -enum vk_mul_mat_id_pin_kernel { VK_MUL_MAT_ID_PIN_KERNEL_MM, VK_MUL_MAT_ID_PIN_KERNEL_VEC }; + const uint32_t masksh = Bc * (Br + 1) * float_type_size; + // DATA_A_IQ4_NL is compiled into the FA shaders unconditionally, so its shared table is always allocated. + const uint32_t iq_shmem = 16 * float_type_size; -static vk_mul_mat_id_pin_kernel ggml_vk_mul_mat_id_pin_kernel() { - static const vk_mul_mat_id_pin_kernel kernel = [] { - const char * env = std::getenv("GGML_VK_MUL_MAT_ID_PIN_KERNEL"); - if (env != nullptr && std::strcmp(env, "vec") == 0) { - return VK_MUL_MAT_ID_PIN_KERNEL_VEC; - } - return VK_MUL_MAT_ID_PIN_KERNEL_MM; - }(); - return kernel; -} + uint32_t Qf, kvsh, kblocksh_size; + if (mmq) { + // block_b_cache: int32_t qs[8] + FLOAT_TYPEV2 ds + const uint32_t block_b_size = 8 * sizeof(int32_t) + 2 * float_type_size; + Qf = Br * (hsk / 32) * block_b_size; -// GGML_VK_MUL_MAT_ID_PIN_TOKENS: the N handed to ggml_vk_guess_matmul_id_pipeline -// for a pinned MUL_MAT_ID, i.e. which matmul-id tile gets picked. 0 or unset = -// "auto" (the estimate below). 512 reproduces the pre-change behaviour exactly. -// Only consulted when the pin hint is set. -static uint32_t ggml_vk_mul_mat_id_pin_tokens_override() { - static const uint32_t value = [] { - const char * env = std::getenv("GGML_VK_MUL_MAT_ID_PIN_TOKENS"); - if (env == nullptr) { - return uint32_t(0); - } - char * end = nullptr; - const long parsed = std::strtol(env, &end, 10); - return (end != env && *end == '\0' && parsed > 0) ? (uint32_t) parsed : uint32_t(0); - }(); - return value; -} + // kvsh uses D = HSV (K goes through kblocksh instead) + kvsh = params.shmem_staging ? Bc * (hsv / 4 + 1) * 4 * float_type_size : 4 * float_type_size; -// How many B rows one expert actually gets, which is what the matmul-id tile's N -// dimension should be sized for -- NOT the token count. The pin hint used to -// force N = GGML_VK_MUL_MAT_ID_FORCE_MM_REFERENCE_TOKENS (512), which always -// selects the large 128x128 tile; with ~7 rows per expert that spends 121/128 of -// every tile column on padding. Total selected rows is nei0*nei1, spread over at -// most n_as experts, so nei0*nei1/n_as is a lower bound on the average. It is a -// lower bound (an arena slab has more slots than are routed to), which biases -// towards the smaller tile; sweep GGML_VK_MUL_MAT_ID_PIN_TOKENS to check. -static uint32_t ggml_vk_mul_mat_id_pin_pipeline_n(uint64_t nei0, uint64_t nei1, uint64_t n_as) { - const uint32_t override_n = ggml_vk_mul_mat_id_pin_tokens_override(); - if (override_n != 0) { - return override_n; - } - if (n_as == 0) { - return GGML_VK_MUL_MAT_ID_FORCE_MM_REFERENCE_TOKENS; + // The mixed MMQ shader uses a superset block_a_cache that fits every + // FA-supported quant: int32_t qs[8] + uint32_t qh + FLOAT_TYPEV2 dm. + // Single-scale types leave dm.y unused; non-Q5_* leave qh unused. + const uint32_t block_a_size = 8 * sizeof(int32_t) + sizeof(uint32_t) + 2 * float_type_size; + kblocksh_size = params.shmem_staging ? Bc * (hsk / 32) * block_a_size : block_a_size; + } else { + Qf = Br * (hsk / 4 + 1) * 4 * float_type_size; + + const uint32_t D = std::max(hsk, hsv); + kvsh = params.shmem_staging ? Bc * (D / 4 + 1) * 4 * float_type_size : 4 * float_type_size; + + kblocksh_size = 0; } - const uint64_t rows_per_expert = CEIL_DIV(nei0 * nei1, n_as); - // Floor of 9, not 1: ggml_vk_mul_mat_id_q_f16 only takes the *aligned* - // pipeline variant when pipeline_n > 8, and dropping to the unaligned - // shader is a separate (slower) change that would confound the tile-size - // measurement. 9..32 all select the small 32x32 tile anyway. - return (uint32_t) std::min<uint64_t>(std::max<uint64_t>(rows_per_expert, 9), - GGML_VK_MUL_MAT_ID_FORCE_MM_REFERENCE_TOKENS); -} -static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) { - VK_LOG_DEBUG("ggml_vk_mul_mat_id_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; - std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; - std::cerr << "), (" << ids << ", name=" << ids->name << ", type=" << ids->type << ", ne0=" << ids->ne[0] << ", ne1=" << ids->ne[1] << ", ne2=" << ids->ne[2] << ", ne3=" << ids->ne[3] << ", nb0=" << ids->nb[0] << ", nb1=" << ids->nb[1] << ", nb2=" << ids->nb[2] << ", nb3=" << ids->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)"); - GGML_ASSERT(ggml_vk_dim01_contiguous(src1) || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); // NOLINT - GGML_ASSERT(ids->type == GGML_TYPE_I32); + const uint32_t total_size = tmpsh + tmpshv4 + masksh + iq_shmem + Qf + kvsh + kblocksh_size; + const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; - const uint64_t ne00 = src0->ne[0]; - const uint64_t ne01 = src0->ne[1]; - const uint64_t ne02 = src0->ne[2]; - // const uint64_t ne03 = src0->ne[3]; + VK_LOG_DEBUG("ggml_vk_flash_attn_scalar_shmem_support(HSK=" << hsk << ", HSV=" << hsv << ", mmq=" << mmq << ", total_size=" << total_size << ", supported=" << supported); - const uint64_t ne10 = src1->ne[0]; - const uint64_t ne11 = src1->ne[1]; - const uint64_t ne12 = src1->ne[2]; - const uint64_t ne13 = src1->ne[3]; - - const uint64_t nei0 = ids->ne[0]; - const uint64_t nei1 = ids->ne[1]; - const bool pinned = ggml_vk_mul_mat_id_hint_pinned(dst); - const bool force_mm = ggml_vk_mul_mat_id_force_mm(ne12) || pinned; - // A pinned op sizes the tile from the per-expert row count, not from the - // token count: see ggml_vk_mul_mat_id_pin_pipeline_n. GGML_MUL_MAT_ID_FORCE_MM - // (the env-driven pin) keeps its 512-token reference so its behaviour is - // untouched; so does every unpinned op. - const uint32_t pipeline_n = pinned ? ggml_vk_mul_mat_id_pin_pipeline_n(nei0, nei1, ne02) : - force_mm ? GGML_VK_MUL_MAT_ID_FORCE_MM_REFERENCE_TOKENS : (uint32_t) nei1; - - const uint32_t nbi0 = ids->nb[0]; - const uint32_t nbi1 = ids->nb[1]; - const uint32_t nbi2 = ids->nb[2]; - - const uint64_t ne20 = dst->ne[0]; - const uint64_t ne21 = dst->ne[1]; - // const uint64_t ne22 = dst->ne[2]; - // const uint64_t ne23 = dst->ne[3]; - - const uint64_t n_as = ne02; - // n_as counts, n_as offsets, one total, then one packed row id per (expert, token). - // Hoisting requires 16-bit indices for the packing and a table that fits one binding. - const uint64_t hoisted_row_id_words = 2 * n_as + 1 + nei0 * nei1; - const bool hoist_row_ids = n_as <= 256 && nei0 <= 0xffff && nei1 <= 0xffff && - hoisted_row_id_words * sizeof(uint32_t) <= - ctx->device->properties.limits.maxStorageBufferRange; + return supported; +} - ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; - ggml_backend_vk_buffer_context * src0_buf_ctx = (ggml_backend_vk_buffer_context *)src0->buffer->context; - ggml_backend_vk_buffer_context * src1_buf_ctx = (ggml_backend_vk_buffer_context *)src1->buffer->context; - ggml_backend_vk_buffer_context * ids_buf_ctx = (ggml_backend_vk_buffer_context *)ids->buffer->context; +bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) { + GGML_UNUSED(v_type); + // Needs to be kept up to date on shader changes + const uint32_t Br = params.block_rows; + const uint32_t Bc = params.block_cols; - vk_buffer d_Qx = nullptr; - size_t qx_buf_offset = 0; - vk_buffer d_Qy = nullptr; - size_t qy_buf_offset = 0; - vk_buffer d_ids = nullptr; - size_t ids_buf_offset = 0; + const uint32_t MatBr = 16, MatBc = 16; - bool src0_uma = false; - bool src1_uma = false; - bool ids_uma = false; + const uint32_t row_split = Bc / MatBc; - if (ctx->device->uma) { - ggml_vk_host_get(ctx->device, src0->data, d_Qx, qx_buf_offset); - ggml_vk_host_get(ctx->device, src1->data, d_Qy, qy_buf_offset); - ggml_vk_host_get(ctx->device, ids->data, d_ids, ids_buf_offset); - src0_uma = d_Qx != nullptr; - src1_uma = d_Qy != nullptr; - ids_uma = d_ids != nullptr; - } + const uint32_t hsk_pad = ROUNDUP_POW2(hsk, 16); + const uint32_t hsv_pad = ROUNDUP_POW2(hsv, 16); - // Reformat and convert to fp16 if non-contiguous, or for coopmat2 for better perf - const bool x_non_contig = (ctx->device->coopmat2 && src0->type == GGML_TYPE_F32) || - !ggml_vk_dim01_contiguous(src0); - // If src0 is BF16, try to use a BF16 x BF16 multiply - ggml_type f16_type = src0->type == GGML_TYPE_BF16 ? GGML_TYPE_BF16 : GGML_TYPE_F16; -#if defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT) - // B must already be, or be convertible to, the matmul B type used by this path. - const bool y_decode_vector_supported = ctx->device->coopmat2_decode_vector && - (f16_type != GGML_TYPE_BF16 || ctx->device->coopmat2_bf16_support) && - (src1->type == GGML_TYPE_F32 || src1->type == f16_type); - // If B is copied to prealloc_y, we can choose a 4-element-aligned row stride. - const bool y_decode_vector_uses_prealloc = !ggml_vk_dim01_contiguous(src1) || src1->type != f16_type; - // Direct B reads are safe only if row starts and the original buffer offset are 4-element aligned. - const bool y_decode_vector_aligned = - (ne10 % 4 == 0) && - (y_decode_vector_uses_prealloc || get_misalign_bytes(ctx, src1) % (4 * ggml_type_size(src1->type)) == 0); - // Stage B only when decode-vector is available and direct B reads would be misaligned. - const bool y_decode_vector_staging = y_decode_vector_supported && !y_decode_vector_aligned; -#else - const bool y_decode_vector_staging = false; -#endif - const bool y_non_contig = y_decode_vector_staging || - (ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) || - (src0->type == GGML_TYPE_BF16 && src1->type != GGML_TYPE_BF16) || - !ggml_vk_dim01_contiguous(src1); + const uint32_t acctype = f32acc ? 4 : 2; + const uint32_t f16vec4 = 8; - const bool y_f32_kernel = src1->type == GGML_TYPE_F32 && !y_non_contig; + const uint32_t tmpsh = (Bc / MatBc) * sizeof(float); + // DATA_A_IQ4_NL is compiled into the FA shaders unconditionally, so its shared table is always allocated. + const uint32_t iq_shmem = 16 * sizeof(ggml_fp16_t); - bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0; + const uint32_t qstride = hsk_pad / 4 + 2; + const uint32_t Qf = Br * qstride * f16vec4; - // Check for mmq first - vk_matmul_pipeline mmp = quantize_y ? ggml_vk_get_mul_mat_mat_id_pipeline(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr; + const uint32_t psh_stride = Br / 4 + 2; + const uint32_t Psh = Bc * psh_stride * f16vec4; - if (mmp == nullptr) { - // Fall back to f16 dequant mul mat - mmp = ggml_vk_get_mul_mat_mat_id_pipeline(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]); - quantize_y = false; - } + const uint32_t sfshstride = (hsk <= 128) ? (Br + 8) : Br; + const uint32_t sfsh = Bc * sfshstride * acctype; - const bool qx_needs_dequant = mmp == nullptr || x_non_contig; - bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig); + const uint32_t kvshstride = (params.shmem_staging ? std::max(hsk_pad, hsv_pad) : MatBr) / 4 + 2; + const uint32_t vsh_stride = MatBc / 4 * row_split; + const uint32_t ksh = ((kvshstride >= vsh_stride) ? (Bc * kvshstride) : (Bc * vsh_stride)) * f16vec4; - if (qx_needs_dequant) { - // Fall back to dequant + f16 mulmat - mmp = ggml_vk_get_mul_mat_mat_id_pipeline(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]); - } + // BF16 PVMat accumulator is f32 (no bf16 accumulator support), so pvsh is vec4 (16 bytes) + const uint32_t pvsh_elem_size = (k_type == GGML_TYPE_BF16) ? 16u : f16vec4; + const uint32_t osh_stride = params.row_split * MatBr / 4; + const uint32_t pvsh = MatBc * osh_stride * pvsh_elem_size; - const ggml_type effective_src1_type = quantize_y ? GGML_TYPE_Q8_1 : (y_f32_kernel ? GGML_TYPE_F32 : src1->type); + const uint32_t slope = Br * acctype; - // note: pipeline_n is nei1 unless the pin path forces a reference token count - // (ggml_vk_mul_mat_id_pin_pipeline_n), which keeps the tile choice stable. - const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_id_pipeline_align(ctx, mmp, ne01, pipeline_n, qx_needs_dequant ? f16_type : src0->type, effective_src1_type)); - // Coopmat2 MUL_MAT_ID BK specialization constants in ggml_vk_load_shaders are at most 64. - const uint32_t y_staged_row_stride = ctx->device->coopmat2 && !quantize_y ? ggml_vk_align_size(ne10, 64) : ne10; - const bool y_needs_k_padding = ne10 != y_staged_row_stride; - const bool y_needs_reformat = y_non_contig || y_needs_k_padding; - qy_needs_dequant = qy_needs_dequant || y_needs_k_padding; + const uint32_t total_size = tmpsh + iq_shmem + Qf + Psh + sfsh + ksh + pvsh + slope; + const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; - // Not implemented - GGML_ASSERT(y_needs_reformat || !qy_needs_dequant); // NOLINT + VK_LOG_DEBUG("ggml_vk_flash_attn_coopmat_shmem_support(HSK=" << hsk << ", HSV=" << hsv << ", f32acc=" << f32acc << ", total_size=" << total_size << ", supported=" << supported); - const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && pipeline_n > 8; + return supported; +} - vk_pipeline pipeline = ggml_vk_guess_matmul_id_pipeline(ctx, mmp, ne01, pipeline_n, aligned, qx_needs_dequant ? f16_type : src0->type, effective_src1_type); +// ---- outlier-matrix sweep (2026-07-01): runtime centroid-table toggle ---- +// +// Vulkan-side equivalent of ggml_cuda_turbo4_64_ol_use_n64_table() in +// turbo-quant.cuh. Temporary, investigative-only mechanism (see task brief +// .superpowers/sdd/outlier-matrix-task-brief.md): GGML_TURBO4_64_OL_TABLE=n64 +// selects the N=64-recalibrated centroid table for the non-outlier elements +// of turbo4_64_ol/ol8/ol12; anything else (including unset) keeps the +// default table. Read once on the host per process and cached, then passed +// down as the use_n64_table push-constant field to the scatter/attn/decode +// pipelines whenever the KV cache type is one of the turbo4_64_ol family. +static bool ggml_vk_turbo4_64_ol_use_n64_table() { + static const bool use_n64 = []() { + const char * v = getenv("GGML_TURBO4_64_OL_TABLE"); + return v != nullptr && strcmp(v, "n64") == 0; + }(); + return use_n64; +} - if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { - pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); - } +// SP2: paged attention (GGML_OP_PAGED_ATTN_MT). Mirrors the CUDA dispatch +// (mt::ggml_cuda_op_paged_attn_mt): scatter K_cur/V_cur into the paged cache, +// barrier, then prefill attention. F16 cache path (Task 3); Task 4 adds turbo4_0. +static void ggml_vk_paged_attn_mt(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k_cache = dst->src[1]; + const ggml_tensor * v_cache = dst->src[2]; + const ggml_tensor * block_tables = dst->src[3]; + const ggml_tensor * context_lens = dst->src[4]; + const ggml_tensor * q_lens = dst->src[5]; + const ggml_tensor * k_cur = dst->src[6]; + const ggml_tensor * v_cur = dst->src[7]; + const ggml_tensor * slot_mapping = dst->src[8]; - // GGML_VK_MUL_MAT_ID_PIN_TRACE=1: which tile a pinned mul_mat_id actually got. - static const bool pin_trace = std::getenv("GGML_VK_MUL_MAT_ID_PIN_TRACE") != nullptr; - if (pin_trace && pinned) { - static int n_traced = 0; - if (n_traced < 8) { - ++n_traced; - fprintf(stderr, "[vk-mmid-pin] mm pipeline=%s wg_denoms={%u,%u} pipeline_n=%u aligned=%d " - "m=%u nei0=%u nei1=%u n_as=%u\n", - pipeline->name.c_str(), pipeline->wg_denoms[0], pipeline->wg_denoms[1], - pipeline_n, (int) aligned, (uint32_t) ne01, - (uint32_t) nei0, (uint32_t) nei1, (uint32_t) ne02); - } - } + // op_params: [0]=float scale, [1]=block_size, [2]=max_blocks_per_seq, [3]=n_kv_heads. + const float * op_params_f = (const float *)dst->op_params; + const float scale = op_params_f[0]; + const uint32_t block_size = ((const int32_t *)dst->op_params)[1]; + const uint32_t max_bps = ((const int32_t *)dst->op_params)[2]; + const uint32_t n_kv_heads = ((const int32_t *)dst->op_params)[3]; - const uint64_t x_ne = ggml_nelements(src0); - const uint64_t y_ne = (uint64_t)y_staged_row_stride * ne11 * ne12 * ne13; - const uint64_t d_ne = ggml_nelements(dst); + const uint32_t head_size = (uint32_t)q->ne[0]; + const uint32_t n_heads = (uint32_t)q->ne[1]; + const uint32_t n_tokens = (uint32_t)q->ne[2]; + const uint32_t n_seq = (uint32_t)block_tables->ne[1]; - const uint64_t src0_type_size = ggml_type_size(src0->type); - const uint64_t src0_blck_size = ggml_blck_size(src0->type); - const uint64_t src0_canonical_nb2 = - (ne00 * ne01 / src0_blck_size) * src0_type_size; - const bool src0_strided = src0->nb[2] != src0_canonical_nb2; - const bool src0_stride_representable = src0_type_size != 0 && - src0_blck_size != 0 && src0->nb[2] % src0_type_size == 0 && - (src0->nb[2] / src0_type_size) <= UINT32_MAX / src0_blck_size; - const bool src0_arena_stride = src0_strided && src0_stride_representable; - // The A-dequant staging step below (qx_needs_dequant) flattens src0 with - // ggml_nelements()/x_ne and writes a DENSE fp16 copy: it has no notion of - // nb[2], so for an arena slab it would decode the wrong bytes for every - // slot past the first, and size prealloc_x by the whole slab. Fail loudly - // instead. This cannot fire while a native mul_mat_id pipeline exists for - // (src0->type, f32/f16), which is the case for every expert weight type. - GGML_ASSERT((!src0_arena_stride || !qx_needs_dequant) && - "arena-strided src0 requires a native mul_mat_id pipeline"); - const uint64_t qx_sz = src0_arena_stride ? ggml_nbytes(src0) : - src0_type_size * x_ne / src0_blck_size; - const uint64_t qy_sz = ggml_type_size(src1->type) * ggml_nelements(src1) / ggml_blck_size(src1->type); - const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne; - const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : (y_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne); - const uint64_t ids_sz = nbi2; - const uint64_t d_sz = sizeof(float) * d_ne; + const ggml_type cache_type = k_cache->type; + vk_pipeline scatter_pipeline = ctx->device->pipeline_paged_attn_scatter[cache_type]; + vk_pipeline attn_pipeline = ctx->device->pipeline_paged_attn[cache_type]; + GGML_ASSERT(scatter_pipeline != nullptr && attn_pipeline != nullptr); - vk_pipeline to_fp16_vk_0 = nullptr; - vk_pipeline to_fp16_vk_1 = nullptr; - vk_pipeline to_q8_1 = nullptr; + // Decode vs prefill routing. Decode (the split-K single-query fast path) is + // valid only when EVERY sequence has q_len == 1. Each seq always has at + // least one query token, so the sum of q_lens (== n_tokens == dst->ne[2]) + // equals n_seq iff every q_len is exactly 1 — a shape-only test, no host + // read of the q_lens tensor (no GPU stall). Anything else falls back to the + // Task 3-4 prefill path, which is always correct. + const bool decode = (n_tokens == n_seq); + vk_pipeline decode_pipeline = ctx->device->pipeline_paged_attn_decode[cache_type]; + vk_pipeline reduce_pipeline = ctx->device->pipeline_paged_attn_decode_reduce; - auto make_y_staged_dst = [&]() { - ggml_tensor y_staged_dst = *src1; - y_staged_dst.type = f16_type; - y_staged_dst.nb[0] = ggml_type_size(f16_type); - y_staged_dst.nb[1] = y_staged_dst.nb[0] * y_staged_row_stride; - y_staged_dst.nb[2] = y_staged_dst.nb[1] * ne11; - y_staged_dst.nb[3] = y_staged_dst.nb[2] * y_staged_dst.ne[2]; - return y_staged_dst; - }; + // Partials buffer stride: one chunk per CHUNK_KV (=128) tokens of allocated + // context (max_blocks_per_seq * block_size is an upper bound on real ctx). + const uint32_t CHUNK_KV = 128u; + const uint32_t num_splits = (max_bps * block_size + CHUNK_KV - 1u) / CHUNK_KV; - if (x_non_contig) { - to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0, nullptr, f16_type); + ggml_pipeline_request_descriptor_sets(ctx, scatter_pipeline, 1); + if (decode) { + GGML_ASSERT(decode_pipeline != nullptr && reduce_pipeline != nullptr); + ggml_pipeline_request_descriptor_sets(ctx, decode_pipeline, 1); + ggml_pipeline_request_descriptor_sets(ctx, reduce_pipeline, 1); } else { - to_fp16_vk_0 = ggml_vk_get_to_fp16(ctx, src0->type); + ggml_pipeline_request_descriptor_sets(ctx, attn_pipeline, 1); } - if (y_needs_reformat) { - ggml_tensor y_staged_dst; - const ggml_tensor * y_staged_dst_ptr = nullptr; - if (y_needs_k_padding) { - y_staged_dst = make_y_staged_dst(); - y_staged_dst_ptr = &y_staged_dst; - } - to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, y_staged_dst_ptr, f16_type); - } else { - to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); - } - GGML_ASSERT(!qx_needs_dequant || to_fp16_vk_0 != nullptr); // NOLINT - GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT + vk_subbuffer q_buf = ggml_vk_tensor_subbuffer(ctx, q); + vk_subbuffer k_cache_buf = ggml_vk_tensor_subbuffer(ctx, k_cache); + vk_subbuffer v_cache_buf = ggml_vk_tensor_subbuffer(ctx, v_cache); + vk_subbuffer block_tables_buf = ggml_vk_tensor_subbuffer(ctx, block_tables); + vk_subbuffer context_lens_buf = ggml_vk_tensor_subbuffer(ctx, context_lens); + vk_subbuffer q_lens_buf = ggml_vk_tensor_subbuffer(ctx, q_lens); + vk_subbuffer k_cur_buf = ggml_vk_tensor_subbuffer(ctx, k_cur); + vk_subbuffer v_cur_buf = ggml_vk_tensor_subbuffer(ctx, v_cur); + vk_subbuffer slot_mapping_buf = ggml_vk_tensor_subbuffer(ctx, slot_mapping); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); - if (quantize_y) { - to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); - } - vk_pipeline count_experts = ctx->device->pipeline_count_experts; + // Phase 1: scatter K_cur/V_cur into the paged cache. + // grid = { n_tokens, n_kv_heads * n_qblk, 2 } (z selects K vs V; y fans out + // per 128-element quant-block so head_dim multiples of 128 are supported). + const bool is_t64_ol_family = (cache_type == GGML_TYPE_TURBO4_64_OL || + cache_type == GGML_TYPE_TURBO4_64_OL8 || + cache_type == GGML_TYPE_TURBO4_64_OL12); + const uint32_t qblk_elems = (cache_type == GGML_TYPE_Q8_0) ? 32u : + (cache_type == GGML_TYPE_TURBO4_64 || is_t64_ol_family) ? 64u : 128u; // q8_0: 32-elt blocks, turbo4_64/turbo4_64_ol family: 64-elt blocks + const uint32_t n_qblk = head_size / qblk_elems; // head_dim is a multiple of qblk_elems + // Outlier-matrix sweep (investigative-only, see ggml_vk_turbo4_64_ol_use_n64_table + // above): only meaningful for the turbo4_64_ol/ol8/ol12 family; 0 (default + // table) for every other cache type. + const uint32_t use_n64_table = (is_t64_ol_family && ggml_vk_turbo4_64_ol_use_n64_table()) ? 1u : 0u; + const vk_op_paged_scatter_pc scatter_pc = { head_size, block_size, n_kv_heads, n_tokens, use_n64_table }; + ggml_vk_dispatch_pipeline(ctx, subctx, scatter_pipeline, + { slot_mapping_buf, k_cur_buf, v_cur_buf, k_cache_buf, v_cache_buf }, + scatter_pc, { n_tokens, n_kv_heads * n_qblk, 2 }); - const size_t expert_data_size = sizeof(uint32_t) * - (hoist_row_ids ? hoisted_row_id_words : n_as); + // Barrier so attention reads the freshly scattered cache. + ggml_vk_sync_buffers(ctx, subctx); - { - if ( - (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) || - (qy_needs_dequant && y_sz > ctx->device->properties.limits.maxStorageBufferRange)) { - GGML_ABORT("Requested preallocation size is too large"); - } - if (qx_needs_dequant && ctx->prealloc_size_x < x_sz) { - ctx->prealloc_size_x = x_sz; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if ((qy_needs_dequant || quantize_y) && ctx->prealloc_size_y < y_sz) { - ctx->prealloc_size_y = y_sz; - ggml_vk_preallocate_buffers(ctx, subctx); + if (decode) { + // Phase 2 (decode): split-K. Pass-1 computes per-(seq, head, chunk) + // partials into a scratch buffer; pass-2 merges them with a log-sum-exp + // combine. Mirrors the flash-attention split-K scratch mechanism: size + // ctx->prealloc_split_k via prealloc_size_split_k and reuse that buffer. + const uint64_t partials_size = + (uint64_t)num_splits * n_seq * n_heads * (head_size + 2) * sizeof(float); + if (partials_size > ctx->device->properties.limits.maxStorageBufferRange) { + GGML_ABORT("paged decode partials buffer too large"); } - if (ctx->prealloc_size_split_k < expert_data_size) { - ctx->prealloc_size_split_k = expert_data_size; + if (ctx->prealloc_size_split_k < partials_size) { + ctx->prealloc_size_split_k = partials_size; ggml_vk_preallocate_buffers(ctx, subctx); } - - // Request descriptor sets - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - if (qx_needs_dequant) { - ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_0, 1); - } - if (qy_needs_dequant) { - ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_1, 1); - } - if (quantize_y) { - ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1); + if (ctx->prealloc_split_k_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); } - ggml_pipeline_request_descriptor_sets(ctx, count_experts, 1); - } + vk_subbuffer partials_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_split_k, 0); - vk_buffer d_D = dst_buf_ctx->dev_buffer; - const uint64_t d_buf_offset = vk_tensor_offset(dst) + dst->view_offs; - GGML_ASSERT(d_D != nullptr); - vk_buffer d_X; - uint64_t x_buf_offset = 0; - vk_buffer d_Y; - uint64_t y_buf_offset = 0; - if (!src0_uma) { - d_Qx = src0_buf_ctx->dev_buffer; - qx_buf_offset = vk_tensor_offset(src0) + src0->view_offs; - GGML_ASSERT(d_Qx != nullptr); - } - if (!src1_uma) { - d_Qy = src1_buf_ctx->dev_buffer; - qy_buf_offset = vk_tensor_offset(src1) + src1->view_offs; - GGML_ASSERT(d_Qy != nullptr); - } - if (!ids_uma) { - d_ids = ids_buf_ctx->dev_buffer; - ids_buf_offset = vk_tensor_offset(ids) + ids->view_offs; - GGML_ASSERT(d_ids != nullptr); - } - if (qx_needs_dequant) { - d_X = ctx->prealloc_x; - GGML_ASSERT(d_X->size >= x_sz); - } else { - d_X = d_Qx; - x_buf_offset = qx_buf_offset; - GGML_ASSERT(qx_sz == x_sz); - } - if (qy_needs_dequant) { - d_Y = ctx->prealloc_y; - GGML_ASSERT(d_Y->size >= y_sz); - } else if (quantize_y) { - d_Y = ctx->prealloc_y; - GGML_ASSERT(d_Y->size >= CEIL_DIV(y_sz, 144) * 144); - } else { - d_Y = d_Qy; - y_buf_offset = qy_buf_offset; - GGML_ASSERT(qy_sz == y_sz); - } + // Pass 1: per-chunk partials. grid = { n_seq, n_heads, num_splits }. + const vk_op_paged_decode_pc decode_pc = { + head_size, block_size, n_heads, n_kv_heads, max_bps, n_seq, num_splits, scale, use_n64_table + }; + ggml_vk_dispatch_pipeline(ctx, subctx, decode_pipeline, + { q_buf, k_cache_buf, v_cache_buf, block_tables_buf, context_lens_buf, q_lens_buf, partials_buf }, + decode_pc, { n_seq, n_heads, num_splits }); - if (x_non_contig || qx_needs_dequant) { - if (ctx->prealloc_x_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - } - // Count how many times each expert is used - vk_subbuffer expert_count_buf = { ctx->prealloc_split_k, 0, expert_data_size }; - if (ctx->prealloc_split_k_need_sync) { + // Barrier so the reduce reads completed partials. ggml_vk_sync_buffers(ctx, subctx); - } - { - vk_op_count_experts_push_constants pc = { (uint32_t)nei0, - (uint32_t)nei1, - (uint32_t)(nbi0 / ggml_type_size(ids->type)), - (uint32_t)(nbi1 / ggml_type_size(ids->type)), - (uint32_t)(get_misalign_bytes(ctx, ids) / ggml_type_size(ids->type)), - (uint32_t)n_as, - uint32_t(hoist_row_ids), - 0, 0 }; - init_pushconst_fastdiv(pc); - ggml_vk_dispatch_pipeline(ctx, subctx, count_experts, - { vk_subbuffer{ d_ids, ids_buf_offset, ids_sz }, expert_count_buf }, pc, - { hoist_row_ids ? 1u : (uint32_t)n_as, 1, 1}); - } - if (x_non_contig) { - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, ggml_vk_subbuffer(ctx, d_Qx, qx_buf_offset), ggml_vk_subbuffer(ctx, d_X, 0)); - } else if (qx_needs_dequant) { - const std::vector<uint32_t> pc = { (uint32_t)ne01, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)(ggml_nelements(src0)) }; - ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, - { vk_subbuffer{ d_Qx, qx_buf_offset, qx_sz }, vk_subbuffer{ d_X, 0, x_sz } }, pc, { (uint32_t)x_ne, 1, 1}); - } - if (y_needs_reformat) { - if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded != y_needs_k_padding) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - if (y_needs_k_padding) { - GGML_ASSERT(y_sz % 4 == 0); - // Zero B padding because clamping only A can produce 0 * Inf or NaN. - subctx->s->buffer->buf.fillBuffer(d_Y->buffer, 0, y_sz, 0); - ggml_vk_sync_buffers(ctx, subctx); - const ggml_tensor y_staged_dst = make_y_staged_dst(); - const uint32_t y_staged_dst_type_size = ggml_type_size(y_staged_dst.type); - ggml_vk_cpy_to_strided( - ctx, subctx, to_fp16_vk_1, src1, - ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), ggml_vk_subbuffer(ctx, d_Y, 0), - (uint32_t)(y_staged_dst.nb[0] / y_staged_dst_type_size), - (uint32_t)(y_staged_dst.nb[1] / y_staged_dst_type_size), - (uint32_t)(y_staged_dst.nb[2] / y_staged_dst_type_size), - (uint32_t)(y_staged_dst.nb[3] / y_staged_dst_type_size)); - } else { - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), ggml_vk_subbuffer(ctx, d_Y, 0)); - } - ctx->prealloc_y_last_pipeline_used = to_fp16_vk_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = y_needs_k_padding; - } - } - if (quantize_y) { - if (ctx->prealloc_y_last_pipeline_used != to_q8_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), ggml_vk_subbuffer(ctx, d_Y, 0), y_ne); - ctx->prealloc_y_last_pipeline_used = to_q8_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; - } - } - ggml_vk_sync_buffers(ctx, subctx); + // Pass 2: reduce. grid = { n_seq, n_heads, 1 }. + const vk_op_paged_decode_reduce_pc reduce_pc = { head_size, n_heads, n_seq, num_splits }; + ggml_vk_dispatch_pipeline(ctx, subctx, reduce_pipeline, + { partials_buf, context_lens_buf, dst_buf }, + reduce_pc, { n_seq, n_heads, 1 }); - uint32_t stride_batch_x = ne00*ne01; - if (src0_arena_stride) { - const uint64_t stride_batch_x64 = - (src0->nb[2] / src0_type_size) * src0_blck_size; - GGML_ASSERT(src0_type_size != 0 && src0->nb[2] % src0_type_size == 0); // NOLINT - GGML_ASSERT(stride_batch_x64 <= UINT32_MAX); // NOLINT - stride_batch_x = (uint32_t) stride_batch_x64; + ctx->prealloc_split_k_need_sync = true; + } else { + // Phase 2: prefill attention. grid = { n_tokens, n_heads, 1 }. + const vk_op_paged_attn_pc attn_pc = { + head_size, block_size, n_heads, n_kv_heads, max_bps, n_seq, n_tokens, scale, use_n64_table + }; + ggml_vk_dispatch_pipeline(ctx, subctx, attn_pipeline, + { q_buf, k_cache_buf, v_cache_buf, block_tables_buf, context_lens_buf, q_lens_buf, dst_buf }, + attn_pc, { n_tokens, n_heads, 1 }); } - uint32_t stride_b_y = y_needs_k_padding ? y_staged_row_stride : ne10; - uint32_t stride_batch_y = y_needs_k_padding ? y_staged_row_stride * ne11 : ne10*ne11; +} - if (!ggml_vk_dim01_contiguous(src0) && !qx_needs_dequant) { - stride_batch_x = src0->nb[0] / ggml_type_size(src0->type); +void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v, const ggml_tensor * mask, const ggml_tensor * sinks, ggml_tensor * dst) { + VK_LOG_DEBUG("ggml_vk_flash_attn((" << q << ", name=" << q->name << ", type=" << q->type << ", ne0=" << q->ne[0] << ", ne1=" << q->ne[1] << ", ne2=" << q->ne[2] << ", ne3=" << q->ne[3] << ", nb0=" << q->nb[0] << ", nb1=" << q->nb[1] << ", nb2=" << q->nb[2] << ", nb3=" << q->nb[3]; + std::cerr << "), (" << k << ", name=" << k->name << ", type=" << k->type << ", ne0=" << k->ne[0] << ", ne1=" << k->ne[1] << ", ne2=" << k->ne[2] << ", ne3=" << k->ne[3] << ", nb0=" << k->nb[0] << ", nb1=" << k->nb[1] << ", nb2=" << k->nb[2] << ", nb3=" << k->nb[3]; + std::cerr << "), (" << v << ", name=" << v->name << ", type=" << v->type << ", ne0=" << v->ne[0] << ", ne1=" << v->ne[1] << ", ne2=" << v->ne[2] << ", ne3=" << v->ne[3] << ", nb0=" << v->nb[0] << ", nb1=" << v->nb[1] << ", nb2=" << v->nb[2] << ", nb3=" << v->nb[3]; + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; + if (sinks) { + std::cerr << "), (" << sinks << ", name=" << sinks->name << ", type=" << sinks->type << ", ne0=" << sinks->ne[0] << ", ne1=" << sinks->ne[1] << ", ne2=" << sinks->ne[2] << ", ne3=" << sinks->ne[3] << ", nb0=" << sinks->nb[0] << ", nb1=" << sinks->nb[1] << ", nb2=" << sinks->nb[2] << ", nb3=" << sinks->nb[3]; } + std::cerr << "))"); - if (!ggml_vk_dim01_contiguous(src1) && !qy_needs_dequant && !quantize_y) { - stride_batch_y = src1->nb[0] / ggml_type_size(src1->type); - } + GGML_TENSOR_LOCALS(int64_t, neq, q, ne) + GGML_TENSOR_LOCALS(size_t, nbq, q, nb) + GGML_TENSOR_LOCALS(int64_t, nek, k, ne) + GGML_TENSOR_LOCALS(size_t, nbk, k, nb) + GGML_TENSOR_LOCALS(int64_t, nev, v, ne) + GGML_TENSOR_LOCALS(size_t, nbv, v, nb) + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) - // Weight paging, prefill path. Same contract as the mul_mat_vec_id path, but - // this shader indexes data_a in units of LOAD_VEC_A elements rather than in - // quant blocks, so republish the offsets in ELEMENTS and let the shader - // divide by LOAD_VEC_A_EFF exactly as it does for batch_stride_a. The block - // count times the block's element count is an exact element offset. - vk_wp_expert_offsets wp_off; - bool wp_paged = ggml_vk_wp_take_expert_offsets(wp_off); - if (wp_paged && qx_needs_dequant) { - // The dequant staging step reads src0 at its single patched slot, so the - // paged offsets cannot be honoured. Fall through to stock addressing: - // wrong for a paged model, but wrong visibly rather than silently. - GGML_LOG_WARN("%s: weight paging: mul_mat_id needs A dequant for %s, paged offsets ignored\n", - __func__, ggml_get_name(src0)); - wp_paged = false; - } - vk_subbuffer d_WP = { d_ids, ids_buf_offset, ids_sz }; // dummy when not paged - vk_buffer x_buf = d_X; - uint64_t x_off = x_buf_offset; - uint64_t x_len = x_sz; - if (wp_paged) { - const uint32_t elems_per_block = (uint32_t) ggml_blck_size(src0->type); - std::vector<uint32_t> elem_off(wp_off.offsets.size()); - for (size_t e = 0; e < elem_off.size(); ++e) { - elem_off[e] = wp_off.offsets[e] * elems_per_block; - } - d_WP = ggml_vk_wp_upload_expert_offsets(ctx, elem_off); - if (wp_off.pool_buffer != nullptr) { - auto * pool_ctx = (ggml_backend_vk_buffer_context *) wp_off.pool_buffer->context; - x_buf = pool_ctx->dev_buffer; - x_off = 0; - x_len = pool_ctx->dev_buffer->size; - } - } + const uint32_t nem0 = mask ? mask->ne[0] : 0; + const uint32_t nem1 = mask ? mask->ne[1] : 0; + const uint32_t nem2 = mask ? mask->ne[2] : 0; + const uint32_t nem3 = mask ? mask->ne[3] : 0; - // compute - ggml_vk_matmul_id( - ctx, subctx, pipeline, - { x_buf, x_off, x_len }, { d_Y, y_buf_offset, y_sz }, - { d_D, d_buf_offset, d_sz }, { d_ids, ids_buf_offset, ids_sz }, expert_count_buf, - d_WP, - ne01, ne21, ne10, ne10, stride_b_y, ne01, - stride_batch_x, stride_batch_y, ne20*ne21, - n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, hoist_row_ids, - wp_paged ? 1u : 0u - ); // NOLINT + const uint32_t HSK = nek0; + const uint32_t HSV = nev0; + uint32_t N = neq1; + const uint32_t KV = nek1; - if (x_non_contig || qx_needs_dequant) { - ctx->prealloc_x_need_sync = true; - } - if (y_needs_reformat || quantize_y) { - ctx->prealloc_y_need_sync = true; - } - ctx->prealloc_split_k_need_sync = true; -} + GGML_ASSERT(ne0 == HSV); + GGML_ASSERT(ne2 == N); -static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { - ggml_tensor * dst = cgraph->nodes[node_idx]; - ggml_tensor * src0 = dst->src[0]; - ggml_tensor * src1 = dst->src[1]; - ggml_tensor * ids = dst->src[2]; - VK_LOG_DEBUG("ggml_vk_mul_mat_vec_id_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; - std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; - std::cerr << "), (" << ids << ", name=" << ids->name << ", type=" << ids->type << ", ne0=" << ids->ne[0] << ", ne1=" << ids->ne[1] << ", ne2=" << ids->ne[2] << ", ne3=" << ids->ne[3] << ", nb0=" << ids->nb[0] << ", nb1=" << ids->nb[1] << ", nb2=" << ids->nb[2] << ", nb3=" << ids->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; - std::cerr << "))"); - GGML_ASSERT(ggml_vk_dim01_contiguous(src0) || src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); // NOLINT - GGML_ASSERT(ggml_vk_dim01_contiguous(src1) || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); // NOLINT - GGML_ASSERT(ids->type == GGML_TYPE_I32); + // input tensor rows must be contiguous + GGML_ASSERT(nbq0 == ggml_type_size(q->type)); + GGML_ASSERT(nbk0 == ggml_type_size(k->type)); + GGML_ASSERT(nbv0 == ggml_type_size(v->type)); - const uint64_t ne00 = src0->ne[0]; - const uint64_t ne01 = src0->ne[1]; - // const uint64_t ne02 = src0->ne[2]; - // const uint64_t ne03 = src0->ne[3]; + GGML_ASSERT(neq0 == HSK); - const uint64_t src0_type_size = ggml_type_size(src0->type); - const uint64_t src0_blck_size = ggml_blck_size(src0->type); - const uint64_t src0_canonical_nb2 = - (ne00 * ne01 / src0_blck_size) * src0_type_size; - const bool src0_strided = src0->nb[2] != src0_canonical_nb2; - const bool src0_stride_representable = src0_type_size != 0 && - src0_blck_size != 0 && src0->nb[2] % src0_type_size == 0 && - (src0->nb[2] / src0_type_size) <= UINT32_MAX / src0_blck_size; - const bool src0_arena_stride = src0_strided && src0_stride_representable; - uint32_t stride_batch_x = (uint32_t)(ne00 * ne01); - if (src0_arena_stride) { - const uint64_t stride_batch_x64 = - (src0->nb[2] / src0_type_size) * src0_blck_size; - GGML_ASSERT(stride_batch_x64 <= UINT32_MAX); // NOLINT - stride_batch_x = (uint32_t) stride_batch_x64; - } + GGML_ASSERT(neq1 == N); - const uint64_t ne10 = src1->ne[0]; - const uint64_t ne11 = src1->ne[1]; - const uint64_t ne12 = src1->ne[2]; - // const uint64_t ne13 = src1->ne[3]; + GGML_ASSERT(nev1 == nek1); - const uint64_t nei0 = ids->ne[0]; - const uint64_t nei1 = ids->ne[1]; - const uint32_t nbi1 = (uint32_t)(ids->nb[1] / sizeof(int)); + // dst cannot be transposed or permuted + GGML_ASSERT(nb0 == sizeof(float)); + GGML_ASSERT(nb0 <= nb1); + GGML_ASSERT(nb1 <= nb2); + GGML_ASSERT(nb2 <= nb3); - const uint64_t ne20 = dst->ne[0]; - const uint64_t ne21 = dst->ne[1]; - // const uint64_t ne22 = dst->ne[2]; - // const uint64_t ne23 = dst->ne[3]; + assert(dst->type == GGML_TYPE_F32); + assert(q->type == GGML_TYPE_F32); + uint32_t gqa_ratio = 1; + uint32_t qk_ratio = neq2 / nek2; + uint32_t workgroups_x = (uint32_t)neq1; + uint32_t workgroups_y = (uint32_t)neq2; + uint32_t workgroups_z = (uint32_t)neq3; - const bool x_non_contig = !ggml_vk_dim01_contiguous(src0); - const bool y_non_contig = !ggml_vk_dim01_contiguous(src1); + const bool f32acc = !ctx->device->fp16 || dst->op_params[3] == GGML_PREC_F32 || k->type == GGML_TYPE_BF16; - const bool f16_f32_kernel = src1->type == GGML_TYPE_F32; - bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type); + // dequant K/V once into an f16 scratch, reordered KV layout so FA can read without a stride + auto is_dense_kv_cache = [](const ggml_tensor * t) { + return t->nb[0] == ggml_type_size(t->type) && + t->nb[2] == ggml_row_size(t->type, t->ne[0]) && + t->nb[1] == t->nb[2] * t->ne[2] && + (t->ne[3] == 1 || t->nb[3] == t->nb[1] * t->ne[1]); + }; + const bool k_quant = k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_BF16 && k->type != GGML_TYPE_F32; + const bool v_quant = v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_BF16 && v->type != GGML_TYPE_F32; + const bool use_dequant_kv = k_quant && v_quant && neq1 >= 64 && + is_dense_kv_cache(k) && is_dense_kv_cache(v) && + (uint64_t)ggml_nelements(k) * sizeof(ggml_fp16_t) <= ctx->device->properties.limits.maxStorageBufferRange && + (uint64_t)ggml_nelements(v) * sizeof(ggml_fp16_t) <= ctx->device->properties.limits.maxStorageBufferRange && + ctx->device->pipeline_dequant_transpose[k->type] != nullptr && + ctx->device->pipeline_dequant_transpose[v->type] != nullptr && + // coopmat2 path does not benefit from the f16 scratch + !ctx->device->coopmat2 && + // Intel Xe1 regresses, see PR 25494 + (ctx->device->vendor_id != VK_VENDOR_ID_INTEL || + (ctx->device->coopmat_support && ctx->device->architecture != vk_device_architecture::INTEL_XE1)); + const ggml_type k_type_eff = use_dequant_kv ? GGML_TYPE_F16 : k->type; + const ggml_type v_type_eff = use_dequant_kv ? GGML_TYPE_F16 : v->type; - vk_pipeline to_fp16_vk_0 = nullptr; - vk_pipeline to_fp16_vk_1 = nullptr; - if (x_non_contig) { - to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0, nullptr, src0->type); - } - if (y_non_contig) { - to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, nullptr, src1->type); - } else { - to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); + // For scalar/coopmat1 FA, we can use the "large" size to accommodate qga. + // For coopmat2 FA, we always use the small size (which is still pretty large for gqa). + vk_fa_tuning_params tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, 512, KV, k_type_eff, v_type_eff, f32acc); + const uint32_t max_gqa = std::min(tuning_params.block_rows, 32u); + + if (N <= 8 && qk_ratio > 1 && qk_ratio <= max_gqa && + qk_ratio * nek2 == neq2 && nek2 == nev2 && nem2 <= 1) { + // grouped query attention - make the N dimension equal to gqa_ratio, reduce + // workgroups proportionally in y dimension. The shader will detect gqa_ratio > 1 + // and change addressing calculations to index Q's dimension 2. + gqa_ratio = qk_ratio; + N = gqa_ratio; + workgroups_y /= gqa_ratio; } - // Check for mmq first - vk_pipeline dmmv = quantize_y ? ggml_vk_get_dequantize_mul_mat_vec_id(ctx, src0->type, GGML_TYPE_Q8_1, ne20, ne00) : nullptr; - vk_pipeline to_q8_1 = nullptr; + tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, N, KV, k_type_eff, v_type_eff, f32acc); - if (dmmv == nullptr) { - // Fall back to f16 dequant mul mat - dmmv = ggml_vk_get_dequantize_mul_mat_vec_id(ctx, src0->type, src1->type, ne20, ne00); - quantize_y = false; + float scale = 1.0f; + float max_bias = 0.0f; + float logit_softcap = 0.0f; + + memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); + memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); + + if (logit_softcap != 0) { + scale /= logit_softcap; } - if (quantize_y) { - to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); + // Sparse mask hint (op_params[4]): compact the <= n_kv_max finite positions and gather only those. + const int32_t n_kv_max = mask ? ggml_get_op_params_i32(dst, 4) : 0; + static const bool disable_sparse = getenv("GGML_VK_FA_SPARSE_DISABLE") != nullptr; + // cm2 dense is fast, so it needs a larger reduction to win. + const int64_t min_ratio = tuning_params.path == FA_COOPMAT2 ? 4 : 2; + const bool use_sparse = !disable_sparse && n_kv_max > 0 && mask && + max_bias == 0.0f && logit_softcap == 0.0f && + k_type_eff == GGML_TYPE_F16 && v_type_eff == GGML_TYPE_F16 && + nem0 == KV && + (int64_t)KV >= std::max<int64_t>(4096, min_ratio * (int64_t)n_kv_max) && + (gqa_ratio > 1 || (tuning_params.path == FA_SCALAR && N == 1)); + + const uint32_t q_stride = (uint32_t)(nbq1 / ggml_type_size(q->type)); + uint32_t k_stride = (uint32_t)(nbk1 / ggml_type_size(k->type)); + uint32_t v_stride = (uint32_t)(nbv1 / ggml_type_size(v->type)); + + // For F32, the shader treats it as a block of size 4 (for vec4 loads) + if (k->type == GGML_TYPE_F32) { + k_stride /= 4; + } + if (v->type == GGML_TYPE_F32) { + v_stride /= 4; } - const bool qx_needs_dequant = x_non_contig; - const bool qy_needs_dequant = !quantize_y && ((src1->type != GGML_TYPE_F16 && !f16_f32_kernel) || y_non_contig); + uint32_t nbk2_eff = (uint32_t)nbk2, nbk3_eff = (uint32_t)nbk3; + uint32_t nbv2_eff = (uint32_t)nbv2, nbv3_eff = (uint32_t)nbv3; + if (use_dequant_kv) { + k_stride = HSK; + v_stride = HSV; + nbk2_eff = (uint32_t)((uint64_t)HSK * KV * sizeof(ggml_fp16_t)); + nbk3_eff = (uint32_t)((uint64_t)HSK * KV * nek2 * sizeof(ggml_fp16_t)); + nbv2_eff = (uint32_t)((uint64_t)HSV * KV * sizeof(ggml_fp16_t)); + nbv3_eff = (uint32_t)((uint64_t)HSV * KV * nev2 * sizeof(ggml_fp16_t)); + } + const uint32_t alignment = tuning_params.block_cols; + bool aligned = (KV % alignment) == 0 && + // the "aligned" shader variant will forcibly align strides, for performance + (q_stride & 7) == 0 && (k_stride & 7) == 0 && (v_stride & 7) == 0; - if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { - dmmv = ggml_vk_get_64b_indexing_pipeline(ctx, dmmv); + // Need to use the coopmat2 variant that clamps loads when HSK/HSV aren't sufficiently aligned. + if (((HSK | HSV) % 16) != 0 && tuning_params.path == FA_COOPMAT2) { + aligned = false; } - // Not implemented - GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT - GGML_ASSERT(!qx_needs_dequant || to_fp16_vk_0 != nullptr); // NOLINT - GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT - GGML_ASSERT(dmmv != nullptr); + // Only use mask opt when the mask is fairly large. This hasn't been tuned extensively. + bool use_mask_opt = mask && !use_sparse && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16 + && (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK > 256 || HSV > 256); + vk_fa_pipeline_state fa_pipeline_state = get_fa_pipeline_state(ctx->device, tuning_params, HSK, HSV, aligned, f32acc, + mask != nullptr, use_mask_opt, logit_softcap != 0, use_sparse, k_type_eff, v_type_eff); - const uint64_t x_ne = ggml_nelements(src0); - const uint64_t y_ne = ggml_nelements(src1); + vk_pipeline pipeline = nullptr; - const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device->properties.limits.minStorageBufferOffsetAlignment); - const uint64_t x_sz = x_non_contig ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : qx_sz; - const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : - (f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne); + bool xe_fa_opt = false; + bool fa_copy_qstate = false; + bool xe_fa_supported_platform = + (ctx->device.get()->architecture == INTEL_XE2 && ctx->device.get()->properties.deviceID != 0xFD80 && ctx->device.get()->properties.deviceID != 0xFD81) || + (ctx->device.get()->architecture == INTEL_XE1 && ctx->device.get()->coopmat_support && ctx->device.get()->uma); + bool xe_fa_supported_usage = neq0 % 32 == 0 && nev0 % 16 == 0 && q->nb[1] > q->nb[2] && k->nb[1] > k->nb[2] && v->nb[1] > v->nb[2] && mask != nullptr; + bool xe_fa_supported_dtype = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16 && (mask != nullptr && mask->type == GGML_TYPE_F16); + std::pair<vk_pipeline, vk_pipeline> xe_fa_pipeline_dual_phases = { nullptr , nullptr }; + vk_pipeline xe_fa_pipeline = nullptr; + size_t size_p = 0; + size_t size_group_max = 0; { - if ( - (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) || - (qy_needs_dequant && y_sz > ctx->device->properties.limits.maxStorageBufferRange)) { - GGML_ABORT("Requested preallocation size is too large"); - } - if (qx_needs_dequant && ctx->prealloc_size_x < x_sz) { - ctx->prealloc_size_x = x_sz; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if ((qy_needs_dequant || quantize_y) && ctx->prealloc_size_y < y_sz) { - ctx->prealloc_size_y = y_sz; - ggml_vk_preallocate_buffers(ctx, subctx); + std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); + auto &pipelines = ctx->device->pipeline_flash_attn_f32_f16; + auto it = pipelines.find(fa_pipeline_state); + if (it != pipelines.end()) { + pipeline = it->second; + } else { + pipelines[fa_pipeline_state] = pipeline = std::make_shared<vk_pipeline_struct>(); } + } - // Request descriptor sets - if (qx_needs_dequant) { - ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_0, 1); - } - if (qy_needs_dequant) { - ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_1, 1); + assert(pipeline); + // Compile early to initialize wg_denoms. + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + + uint32_t split_kv = KV; + uint32_t split_k = 1; + + // Intel Alchemist prefers more workgroups + const uint32_t shader_core_count_multiplier = (ctx->device->vendor_id == VK_VENDOR_ID_INTEL && ctx->device->architecture != INTEL_XE2) ? 2 : 1; + + // Use a placeholder core count if one isn't available. split_k is a big help for perf. + const uint32_t shader_core_count = ctx->device->shader_core_count ? ctx->device->shader_core_count * shader_core_count_multiplier : 16; + + const uint32_t Br = fa_pipeline_state.Br; + const uint32_t Bc = fa_pipeline_state.Bc; + + GGML_ASSERT(Br == pipeline->wg_denoms[0]); + const uint32_t Tr = CEIL_DIV(N, Br); + + // Try to use split_k when KV is large enough to be worth the overhead. + // Sparse: split_kv carries n_kv_max, split_k partitions its blocks for occupancy. + if (use_sparse) { + split_kv = (uint32_t)n_kv_max; + const uint32_t total_blocks = CEIL_DIV((uint32_t)n_kv_max, Bc); + const uint32_t base_wgs = (gqa_ratio > 1 ? workgroups_x : Tr) * workgroups_y * workgroups_z; + if (base_wgs < shader_core_count * 2) { + split_k = shader_core_count * 2 / base_wgs; + } + split_k = std::max(1u, std::min(split_k, total_blocks)); + // Match the shader's per-split block count so no split is empty. + const uint32_t per_blocks = CEIL_DIV(total_blocks, split_k); + split_k = CEIL_DIV(total_blocks, per_blocks); + } else if (gqa_ratio > 1 && workgroups_x <= Br) { + split_k = shader_core_count * 2 / (workgroups_x * workgroups_y * workgroups_z); + } else if (gqa_ratio <= 1) { + uint32_t total_wgs_no_split = Tr * workgroups_y * workgroups_z; + if (total_wgs_no_split < shader_core_count * 2) { + split_k = shader_core_count * 2 / total_wgs_no_split; } - if (quantize_y) { - ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1); + } + + if (!use_sparse && split_k > 1) { + // Try to evenly split KV into split_k chunks, but it needs to be a multiple + // of "align", so recompute split_k based on that. + split_kv = ROUNDUP_POW2(std::max(1u, KV / split_k), alignment); + split_k = CEIL_DIV(KV, split_kv); + xe_fa_opt = xe_fa_supported_platform && xe_fa_supported_usage && xe_fa_supported_dtype; + if (xe_fa_opt) { + std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); + const uint32_t split_p_size = 32; + const size_t max_dim = (nek1 + split_p_size - 1) / split_p_size; + const size_t p_dim = max_dim * split_p_size; + auto& pipelines = ctx->device->pipeline_xe_fa_decode_dual_phases; + auto it = pipelines.find({ (uint32_t)neq0, (uint32_t)nev0, qk_ratio, (uint32_t)neq1 }); + if (it != pipelines.end()) { + xe_fa_pipeline_dual_phases = it->second; + } else { + pipelines[{(uint32_t)neq0, (uint32_t)nev0, qk_ratio, (uint32_t)neq1}] = xe_fa_pipeline_dual_phases = std::make_pair(std::make_shared<vk_pipeline_struct>(), std::make_shared<vk_pipeline_struct>()); + } + + size_p = neq1 * neq2 * p_dim * neq3 * sizeof(ggml_fp16_t); + size_group_max = neq1 * neq2 * max_dim * neq3 * sizeof(float); + size_t temp_size = ggml_nelements(q) * sizeof(ggml_fp16_t) + size_p + size_group_max; + fa_copy_qstate = true; + if (ctx->prealloc_size_x < temp_size) { + ctx->prealloc_size_x = temp_size; + ggml_vk_preallocate_buffers(ctx, subctx); + } + + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } } - ggml_pipeline_request_descriptor_sets(ctx, dmmv, nei1); } - vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, cgraph->nodes[node_idx + ctx->num_additional_fused_ops]); - vk_subbuffer d_Qx = ggml_vk_tensor_subbuffer(ctx, src0); - vk_subbuffer d_Qy = ggml_vk_tensor_subbuffer(ctx, src1); - vk_subbuffer d_ids = ggml_vk_tensor_subbuffer(ctx, ids); - vk_subbuffer d_F0 = d_D; - vk_subbuffer d_X, d_Y; + if (xe_fa_opt == true) { + use_mask_opt = false; + } - if (qx_needs_dequant) { - d_X = { ctx->prealloc_x, 0, ctx->prealloc_x->size }; - } else { - d_X = d_Qx; + // Reserve space for split_k temporaries. For each split x batch, we need to store the O matrix (D x ne1) + // and the per-row m and L values (ne1 rows). We store all the matrices first, followed by the rows. + // For matrices, the order is (inner to outer) [HSV, ne1, k, ne2, ne3]. + // For L/M, the order is (inner to outer) [ne1, k, ne2, ne3]. + const uint64_t split_k_size = split_k > 1 ? (HSV * ne1 * sizeof(float) + ne1 * sizeof(float) * 2) * split_k * ne2 * ne3 : 0; + if (split_k_size > ctx->device->properties.limits.maxStorageBufferRange) { + GGML_ABORT("Requested preallocation size is too large"); } - if (qy_needs_dequant || quantize_y) { - d_Y = { ctx->prealloc_y, 0, ctx->prealloc_y->size }; - } else { - d_Y = d_Qy; + if (ctx->prealloc_size_split_k < split_k_size) { + ctx->prealloc_size_split_k = split_k_size; + ggml_vk_preallocate_buffers(ctx, subctx); } - if (x_non_contig) { - if (ctx->prealloc_x_need_sync) { + const uint32_t mask_opt_num_dwords = CEIL_DIV(nem0, 16 * Bc); + const uint64_t mask_opt_size = sizeof(uint32_t) * mask_opt_num_dwords * CEIL_DIV(nem1, Br) * nem2 * nem3; + + vk_pipeline pipeline_fa_mask_opt = nullptr; + if (use_mask_opt) { + { + std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); + auto &pipelines = ctx->device->pipeline_fa_mask_opt; + auto it = pipelines.find({Br, Bc}); + if (it != pipelines.end()) { + pipeline_fa_mask_opt = it->second; + } else { + pipelines[{Br, Bc}] = pipeline_fa_mask_opt = std::make_shared<vk_pipeline_struct>(); + } + } + assert(pipeline_fa_mask_opt); + ggml_pipeline_request_descriptor_sets(ctx, pipeline_fa_mask_opt, 1); + + if (ctx->prealloc_size_y < mask_opt_size) { + ctx->prealloc_size_y = mask_opt_size; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if (ctx->prealloc_y_need_sync) { ggml_vk_sync_buffers(ctx, subctx); } } - if (x_non_contig) { - GGML_ASSERT(x_sz == ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment)); - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, d_Qx, d_X); - } - if (y_non_contig) { - GGML_ASSERT(y_sz == ggml_type_size(src1->type) * y_ne); - if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, d_Qy, d_Y); - ctx->prealloc_y_last_pipeline_used = to_fp16_vk_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; + // Sparse index scratch reuses prealloc_y (mutually exclusive with mask opt). + const uint64_t sparse_idx_size = use_sparse + ? sizeof(int32_t) * (uint64_t)n_kv_max * nem1 * nem2 * nem3 + : 0; + vk_pipeline sparse_compact_pipeline = ctx->device->fa_sparse_compact_use_subgroups + ? ctx->device->pipeline_fa_sparse_compact_subgroup + : ctx->device->pipeline_fa_sparse_compact; + if (use_sparse) { + ggml_pipeline_request_descriptor_sets(ctx, sparse_compact_pipeline, 1); + if (ctx->prealloc_size_y < sparse_idx_size) { + ctx->prealloc_size_y = sparse_idx_size; + ggml_vk_preallocate_buffers(ctx, subctx); } - } - if (quantize_y) { - if (ctx->prealloc_y_last_pipeline_used != to_q8_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - ggml_vk_quantize_q8_1(ctx, subctx, d_Qy, d_Y, y_ne); - ctx->prealloc_y_last_pipeline_used = to_q8_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; + if (ctx->prealloc_y_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); } } - uint32_t stride_batch_y = ne10*ne11; + const uint32_t n_head_kv = neq2; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv)); + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - if (!ggml_vk_dim01_contiguous(src1) && !qy_needs_dequant) { - stride_batch_y = src1->nb[2] / ggml_type_size(src1->type); + vk_subbuffer q_buf = ggml_vk_tensor_subbuffer(ctx, q); + vk_subbuffer k_buf = ggml_vk_tensor_subbuffer(ctx, k); + vk_subbuffer v_buf = ggml_vk_tensor_subbuffer(ctx, v); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); + vk_subbuffer mask_buf = mask ? ggml_vk_tensor_subbuffer(ctx, mask) : q_buf; + vk_subbuffer sinks_buf = sinks ? ggml_vk_tensor_subbuffer(ctx, sinks) : q_buf; + vk_subbuffer mask_opt_buf = use_mask_opt ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf; + vk_subbuffer sparse_buf = use_sparse ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf; + + if (use_dequant_kv) { + const uint64_t fp = sizeof(ggml_fp16_t); + const uint64_t k_f16_sz = (uint64_t)ggml_nelements(k) * fp; + const uint64_t v_f16_sz = (uint64_t)ggml_nelements(v) * fp; + if (ctx->prealloc_size_x < k_f16_sz + v_f16_sz) { + ctx->prealloc_size_x = k_f16_sz + v_f16_sz; + ggml_vk_preallocate_buffers(ctx, subctx); + } + vk_pipeline tr_k = ctx->device->pipeline_dequant_transpose[k->type]; + vk_pipeline tr_v = ctx->device->pipeline_dequant_transpose[v->type]; + ggml_pipeline_request_descriptor_sets(ctx, tr_k, 1); + ggml_pipeline_request_descriptor_sets(ctx, tr_v, 1); + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + vk_subbuffer k_dst = vk_subbuffer{ ctx->prealloc_x, 0, k_f16_sz }; + vk_subbuffer v_dst = vk_subbuffer{ ctx->prealloc_x, k_f16_sz, v_f16_sz }; + const uint32_t k_nel = (uint32_t)ggml_nelements(k); + const uint32_t v_nel = (uint32_t)ggml_nelements(v); + { const std::vector<uint32_t> pc = { (uint32_t)HSK, (uint32_t)nek2, (uint32_t)KV, 0, k_nel }; + ggml_vk_dispatch_pipeline(ctx, subctx, tr_k, { k_buf, k_dst }, pc, { k_nel, 1, 1 }); } + { const std::vector<uint32_t> pc = { (uint32_t)HSV, (uint32_t)nev2, (uint32_t)KV, 0, v_nel }; + ggml_vk_dispatch_pipeline(ctx, subctx, tr_v, { v_buf, v_dst }, pc, { v_nel, 1, 1 }); } + ggml_vk_sync_buffers(ctx, subctx); + k_buf = k_dst; + v_buf = v_dst; } - const uint32_t max_groups_x = ctx->device->properties.limits.maxComputeWorkGroupCount[0]; + uint32_t mask_n_head_log2 = ((sinks != nullptr) << 24) | n_head_log2; - uint32_t groups_x = ne01; - uint32_t groups_z = 1; + if (use_mask_opt) + { + const vk_op_flash_attn_mask_opt_push_constants opt_pc = { + nem0, + nem1, + nem2, + (uint32_t)(mask->nb[1] / sizeof(ggml_fp16_t)), + (uint32_t)(mask->nb[2] / sizeof(ggml_fp16_t)), + (uint32_t)(mask->nb[3] / sizeof(ggml_fp16_t)), + mask_opt_num_dwords, + mask_opt_num_dwords * CEIL_DIV(nem1, Br), + mask_opt_num_dwords * CEIL_DIV(nem1, Br) * nem2, + }; - if (ne01 > max_groups_x) { - groups_z = 64; - groups_x = CEIL_DIV(groups_x, groups_z); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline_fa_mask_opt, + { mask_buf, mask_opt_buf }, opt_pc, + { mask_opt_num_dwords, CEIL_DIV(nem1, Br), nem2 * nem3 }); + ggml_vk_sync_buffers(ctx, subctx); } - uint32_t fusion_flags = 0; + if (use_sparse) + { + const vk_op_flash_attn_sparse_compact_push_constants sc_pc = { + KV, + nem1, + nem2, + (uint32_t)(mask->nb[1] / sizeof(ggml_fp16_t)), + (uint32_t)(mask->nb[2] / sizeof(ggml_fp16_t)), + (uint32_t)(mask->nb[3] / sizeof(ggml_fp16_t)), + (uint32_t)n_kv_max, + }; - if (ctx->num_additional_fused_ops > 0) { - const ggml_tensor * bias = cgraph->nodes[node_idx + 1]->src[1]; + ggml_vk_dispatch_pipeline(ctx, subctx, sparse_compact_pipeline, + { mask_buf, sparse_buf }, sc_pc, + { nem1, nem2, nem3 }); + ggml_vk_sync_buffers(ctx, subctx); + } - d_F0 = ggml_vk_tensor_subbuffer(ctx, bias); + const vk_flash_attn_push_constants pc = { N, KV, + (uint32_t)ne1, (uint32_t)ne2, (uint32_t)ne3, + (uint32_t)neq2, (uint32_t)neq3, + (uint32_t)nek2, (uint32_t)nek3, + (uint32_t)nev2, (uint32_t)nev3, + nem1, nem2, nem3, + q_stride, (uint32_t)nbq2, (uint32_t)nbq3, + k_stride, nbk2_eff, nbk3_eff, + v_stride, nbv2_eff, nbv3_eff, + scale, max_bias, logit_softcap, + mask_n_head_log2, m0, m1, + gqa_ratio, split_kv, split_k }; - if (cgraph->nodes[node_idx + 1]->op == GGML_OP_MUL) { - fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE0; - } else { - GGML_ASSERT(cgraph->nodes[node_idx + 1]->op == GGML_OP_ADD_ID); - fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; + if (xe_fa_opt && split_k > 1) { + auto upper_power_of_2 = [&](uint32_t in) { + GGML_ASSERT(in != 0); + if (in <= 1) return 1u; + uint32_t ret = in - 1; + ret |= ret >> 1; + ret |= ret >> 2; + ret |= ret >> 4; + ret |= ret >> 8; + ret |= ret >> 16; + return ret + 1; + }; + auto to_fp16_vk_0 = ggml_vk_get_to_fp16(ctx, q->type); + const uint32_t out_dim_per_wg = qk_ratio > 16 ? 8 : 16; + size_t x_ne = ggml_nelements(q); + size_t temp_buf_offset = 0; + uint32_t head_stride_k = uint32_t(nbk2 / ggml_type_size(k->type)); + uint32_t head_stride_v = uint32_t(nbv2 / ggml_type_size(v->type)); + uint32_t batch_stride_q = uint32_t(nbq3 / ggml_type_size(q->type)); + uint32_t batch_stride_k = uint32_t(nbk3 / ggml_type_size(k->type)); + uint32_t batch_stride_v = uint32_t(nbv3 / ggml_type_size(v->type)); + uint32_t batch_stride_m = mask ? uint32_t(mask->nb[3] / ggml_type_size(mask->type)) : 0u; + uint32_t batch_stride_o = uint32_t(nb3 / ggml_type_size(dst->type)); + vk_fa_xe_opt_push_constants pc_ph1 = { (uint32_t)nek1, (uint32_t)neq1, (uint32_t)neq2, (uint32_t)nek2, qk_ratio, 1, (sinks != nullptr) ? 1u : 0u, (uint32_t)k_stride, head_stride_k, + batch_stride_q, batch_stride_k, batch_stride_v, batch_stride_m, batch_stride_o, scale }; + vk_fa_xe_opt_push_constants pc_ph2 = pc_ph1; + pc_ph2.nbkv_tok = v_stride; + pc_ph2.nbkv_head = head_stride_v; + vk_subbuffer q_temp_buf = fa_copy_qstate ? ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset) : q_buf; + temp_buf_offset += fa_copy_qstate ? x_ne * sizeof(ggml_fp16_t) : 0; + vk_subbuffer p_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); + temp_buf_offset += size_p; + vk_subbuffer max_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); + temp_buf_offset += size_group_max; + uint32_t xe_native_sub_group_size = ctx->device.get()->architecture == INTEL_XE1 ? 8 : 16; + uint32_t aligned_gqa_ratio = upper_power_of_2(qk_ratio); + uint32_t out_per_wg_ph1 = std::min(256u / xe_native_sub_group_size, (uint32_t)neq1); + uint32_t out_per_wg_ph2 = std::min(std::max(16u / aligned_gqa_ratio, 1u), (uint32_t)neq1); + uint32_t ph1_wg = ((neq1 + out_per_wg_ph1 - 1) / out_per_wg_ph1) * nek2; + uint32_t ph2_wg = ((neq1 + out_per_wg_ph2 - 1) / out_per_wg_ph2) * ne0 / out_dim_per_wg; + if (fa_copy_qstate) { + const std::vector<uint32_t> pc_cpy_fp16 = + { (uint32_t)q->ne[0], (uint32_t)q->ne[1], (uint32_t)q->ne[2], (uint32_t)q->ne[3], (uint32_t)(x_ne) }; + ggml_vk_sync_buffers(ctx, subctx); + ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_0, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, { q_buf, q_temp_buf }, pc_cpy_fp16, { (uint32_t)(x_ne), 1, 1 }); } - } - vk_subbuffer d_F1 = d_D; - if (ctx->num_additional_fused_ops > 1) { - const ggml_tensor * scale = cgraph->nodes[node_idx + 2]->src[1]; + ggml_vk_sync_buffers(ctx, subctx); + ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline_dual_phases.first, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline_dual_phases.first, + { q_temp_buf, k_buf, mask_buf, p_temp_buf, max_temp_buf }, + pc_ph1, { (uint32_t)ph1_wg, (uint32_t)nek1, (uint32_t)neq3 }); - d_F1 = ggml_vk_tensor_subbuffer(ctx, scale); - fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE1; - } + ggml_vk_sync_buffers(ctx, subctx); + ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline_dual_phases.second, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline_dual_phases.second, + { p_temp_buf, v_buf, max_temp_buf, sinks_buf, dst_buf }, + pc_ph2, { (uint32_t)ph2_wg, (uint32_t)nev2, (uint32_t)neq3 }); - // Weight paging: if the pager published offsets for this node, the experts - // live at arbitrary slots in its pool rather than at a uniform stride in - // src0, so bind the pool and hand the shader the per-slot offsets. Only - // applies when the active-expert count fits the push constants; otherwise - // fall through to stock addressing, which for a paged model is wrong but - // visibly so rather than silently. - vk_wp_expert_offsets wp_off; - bool wp_paged = ggml_vk_wp_take_expert_offsets(wp_off); + ctx->prealloc_x_need_sync = true; + } else if (split_k > 1) { + ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_flash_attn_split_k_reduce, 1); - // GGML_VK_WP_COVERAGE=1: is EVERY expert matmul actually getting a - // publication? An expert node with no publication silently falls through to - // stock uniform-stride addressing, which is wrong for a paged model because - // src0->data has been repointed at a single pool slot. The existing TRACE - // only prints the first 8 dispatches and cannot see this. - static const bool wp_coverage = getenv("GGML_VK_WP_COVERAGE") != nullptr; - if (wp_coverage) { - static size_t n_total = 0, n_paged = 0, n_unpaged_exps = 0; - ++n_total; - if (wp_paged) { - ++n_paged; - } else { - const char * nm = ggml_get_name(src0); - if (nm != nullptr && strstr(nm, "exps") != nullptr) { - if (n_unpaged_exps < 8) { - fprintf(stderr, "[vk-wp-coverage] UNPAGED expert matmul src0=%s\n", nm); - } - ++n_unpaged_exps; - } + if (ctx->prealloc_split_k_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); } - if (n_total % 100 == 0) { - fprintf(stderr, "[vk-wp-coverage] mul_mat_id dispatches=%zu paged=%zu " - "unpaged_exps=%zu\n", n_total, n_paged, n_unpaged_exps); + + // We reuse workgroups_x to mean the number of splits, so we need to + // cancel out the divide by wg_denoms[0]. + uint32_t dispatch_x; + if (gqa_ratio > 1) { + workgroups_x *= pipeline->wg_denoms[0]; + dispatch_x = split_k * workgroups_x; + } else { + dispatch_x = Tr * split_k * pipeline->wg_denoms[0]; } - } - // GGML_VK_WP_FORCE=1: exercise the paged shader path on a NON-paged model, - // publishing offsets that reproduce the stock stride and leaving data_a on - // src0. Correct output here proves the flag/SSBO/shader plumbing; garbage - // proves the plumbing itself is broken, independent of the pager. - static const bool wp_trace = getenv("GGML_VK_WP_TRACE") != nullptr; - if (wp_trace) { - static int n = 0; - if (n < 8) { - ++n; - fprintf(stderr, "[vk-wp-trace] dispatch src0=%-28s paged=%d off0=%u\n", - ggml_get_name(src0), (int) wp_paged, - wp_off.offsets.empty() ? 0u : wp_off.offsets[0]); + + vk_subbuffer split_k_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_split_k, 0); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {q_buf, k_buf, v_buf, mask_buf, sinks_buf, split_k_buf, mask_opt_buf, sparse_buf}, + pc, { dispatch_x, workgroups_y, workgroups_z }); + + ggml_vk_sync_buffers(ctx, subctx); + const vk_op_flash_attn_split_k_reduce_push_constants pc2 = { HSV, (uint32_t)ne1, (uint32_t)ne2, (uint32_t)ne3, split_k, (sinks != nullptr) }; + ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_flash_attn_split_k_reduce, + {split_k_buf, sinks_buf, dst_buf}, + pc2, { (uint32_t)ne1, HSV, (uint32_t)(ne2 * ne3) }); + ctx->prealloc_split_k_need_sync = true; + } else { + if (gqa_ratio > 1) { + // When using gqa, we want one actual workgroup per batch, so cancel out wg_denoms + workgroups_x *= pipeline->wg_denoms[0]; } + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {q_buf, k_buf, v_buf, mask_buf, sinks_buf, dst_buf, mask_opt_buf, sparse_buf}, + pc, { workgroups_x, workgroups_y, workgroups_z }); } - static const bool wp_force = getenv("GGML_VK_WP_FORCE") != nullptr; - if (wp_force && !wp_paged) { - const uint32_t blocks_per_expert = - (uint32_t) ((ne00 * ne01) / ggml_blck_size(src0->type)); - wp_off.offsets.resize((size_t) src0->ne[2]); - for (size_t e = 0; e < wp_off.offsets.size(); ++e) { - wp_off.offsets[e] = (uint32_t) e * blocks_per_expert; - } - wp_off.pool_buffer = nullptr; - wp_paged = true; + + if (use_dequant_kv) { + ctx->prealloc_x_need_sync = true; } - vk_subbuffer d_WP = d_ids; // dummy binding when not paged - // GGML_VK_WP_STOCKOFF=1: keep the paged plumbing (flag + SSBO) but publish - // offsets that reproduce the stock uniform stride and leave data_a on src0. - // Isolates "is the SSBO/flag/shader path correct" from "are the pool - // offsets/binding correct" -- the two halves fail identically otherwise. - static const bool wp_stockoff = getenv("GGML_VK_WP_STOCKOFF") != nullptr; - if (wp_paged && wp_stockoff) { - const uint32_t blocks_per_expert = - (uint32_t) ((ne00 * ne01) / ggml_blck_size(src0->type)); - for (size_t e = 0; e < wp_off.offsets.size(); ++e) { - wp_off.offsets[e] = (uint32_t) e * blocks_per_expert; - } + if (use_mask_opt || use_sparse) { + ctx->prealloc_y_need_sync = true; } - // Deliberately NOT provided: knobs that perturb the published offsets to see - // whether output changes (collapse-to-one-slot, rotate-by-one). They read as - // sensitivity tests but are near no-ops on the paged path, because inactive - // experts already share the first_active_slot fallback and the ACTIVE experts - // keep their own correct offsets. Both produce "no change", which invites the - // false conclusion that the shader is ignoring the pool. Use WP_IDCHK and - // WP_EXPDUMP instead: they test expert IDENTITY, which is what offset-keyed - // checks are blind to. - if (wp_paged && !wp_stockoff && wp_off.pool_buffer != nullptr) { - auto * pool_ctx = (ggml_backend_vk_buffer_context *) wp_off.pool_buffer->context; - d_X = vk_subbuffer{ pool_ctx->dev_buffer, 0, pool_ctx->dev_buffer->size }; +} - // GGML_VK_WP_BIND=1: dump what is actually bound as data_a. The shader - // honours p.paged and data_wp_expert_off (proved by FORCE), the offsets - // are exact and the bytes at them are correct, so the remaining question - // is whether binding 0 is really the pool at dispatch time. - static const bool wp_bind = getenv("GGML_VK_WP_BIND") != nullptr; - if (wp_bind) { - static int n = 0; - if (n < 6) { - ++n; - uint32_t first = 0; - for (uint32_t o : wp_off.offsets) { if (o != 0) { first = o; break; } } - const size_t blk = ggml_type_size(src0->type); - fprintf(stderr, - "[vk-wp-bind] src0=%-26s type=%s blk=%zu\n" - " pool_vkbuf=%p pool_size=%zu\n" - " d_X.vkbuf=%p d_X.off=%zu d_X.size=%zu\n" - " src0_vkbuf=%p src0_off=%zu first_off_blocks=%u byte=%zu\n", - ggml_get_name(src0), ggml_type_name(src0->type), blk, - (void *) (VkBuffer) pool_ctx->dev_buffer->buffer, - (size_t) pool_ctx->dev_buffer->size, - (void *) (VkBuffer) d_X.buffer->buffer, - (size_t) d_X.offset, (size_t) d_X.size, - (void *) (VkBuffer) d_Qx.buffer->buffer, (size_t) d_Qx.offset, - first, (size_t) first * blk); - } - } +static vk_conv_shapes ggml_vk_conv_select_shape(ggml_backend_vk_context * ctx, uint32_t K, uint32_t NPQ) { + auto n_tiles = [&](vk_conv_shapes s) { + return CEIL_DIV(K, vk_conv_block_sizes[s].K) + * CEIL_DIV(NPQ, vk_conv_block_sizes[s].NPQ); + }; - // GGML_VK_WP_HASHCHK=1: re-read the pool at the offsets we are about to - // hand the shader and compare with what stage_in put there. Proves or - // disproves slot-lifetime corruption at the point of use. - if (wp_hashchk_enabled()) { - static size_t n_checked = 0, n_bad = 0, n_untracked = 0, n_dispatch = 0; - const size_t blk = ggml_type_size(src0->type); - if (blk != 0 && ++n_dispatch <= 400) { - std::vector<uint8_t> back(WP_HASH_SPAN); - std::set<uint32_t> seen; - for (uint32_t off_blocks : wp_off.offsets) { - if (!seen.insert(off_blocks).second) { continue; } - const size_t byte_off = (size_t) off_blocks * blk; - if (byte_off + WP_HASH_SPAN > pool_ctx->dev_buffer->size) { continue; } - uint64_t want = 0; - { - std::lock_guard<std::mutex> hg(g_wp_hash_mutex); - auto it = g_wp_slot_hash.find(byte_off); - if (it == g_wp_slot_hash.end()) { ++n_untracked; continue; } - want = it->second; - } - ggml_vk_buffer_read(pool_ctx->dev_buffer, byte_off, back.data(), WP_HASH_SPAN); - const uint64_t got = wp_hash_bytes(back.data(), WP_HASH_SPAN); - ++n_checked; - if (got != want) { - if (n_bad < 8) { - fprintf(stderr, "[vk-wp-hashchk] MISMATCH at pool byte offset %zu " - "(blocks %u, blk %zu) src0=%s\n", - byte_off, off_blocks, blk, ggml_get_name(src0)); - } - ++n_bad; - } - } - if (n_dispatch % 100 == 0) { - fprintf(stderr, "[vk-wp-hashchk] dispatches=%zu offsets_checked=%zu " - "mismatched=%zu untracked=%zu\n", - n_dispatch, n_checked, n_bad, n_untracked); - } - } - } + // We can't query number of shader cores on Intel, use 32 as a placeholder + // so small convolutions will still choose a smaller tile. + const uint32_t shader_core_count = ctx->device->shader_core_count > 0 ? ctx->device->shader_core_count : 32; + + // 128x128 isn't used with cm1 due to shared memory size; fall through to a smaller tile. + bool allow_128x128 = true; +#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + if (!ctx->device->coopmat2 && ctx->device->coopmat_support && ctx->device->coopmat_support_16x16x16_f16acc) { + allow_128x128 = false; } - if (wp_paged) { - d_WP = ggml_vk_wp_upload_expert_offsets(ctx, wp_off.offsets); +#endif + + if (allow_128x128 && K > 64 && n_tiles(CONV_SHAPE_128x128) >= shader_core_count * 2) { + return CONV_SHAPE_128x128; + } else if (K <= 32 && n_tiles(CONV_SHAPE_32x256) >= shader_core_count * 2) { + return CONV_SHAPE_32x256; + } else if (K <= 64 && n_tiles(CONV_SHAPE_64x128) >= shader_core_count * 2) { + return CONV_SHAPE_64x128; + } else if (!allow_128x128 && K > 64 && n_tiles(CONV_SHAPE_64x128) >= shader_core_count * 2) { + // cm1 fallback for large K when 128x128 isn't available + return CONV_SHAPE_64x128; + } else { + return CONV_SHAPE_64x32; } +} - // GGML_VK_WP_IDCHK=1: the general identity invariant. Two DISTINCT experts - // selected by the ids tensor can never legitimately live at the same pool - // offset. If they do, at least one of them was never staged and is silently - // reading the first_active_slot fallback -- which is exactly what a - // stale/mismatched routing readback in the eval callback would produce, and - // which every offset-keyed check is blind to. - static const bool wp_idchk = getenv("GGML_VK_WP_IDCHK") != nullptr; - if (wp_paged && wp_idchk) { - static size_t n_disp = 0, n_viol = 0, n_dup_pairs = 0; - ++n_disp; - const size_t n_ids = (size_t) nei0 * (size_t) nei1; - if (n_ids > 0 && n_ids < 4096) { - std::vector<int32_t> ids_host(n_ids); - ggml_vk_buffer_read(d_ids.buffer, d_ids.offset, - ids_host.data(), n_ids * sizeof(int32_t)); - std::map<uint32_t, int32_t> off_to_id; - bool viol = false; - for (int32_t id : ids_host) { - if (id < 0 || (size_t) id >= wp_off.offsets.size()) { continue; } - const uint32_t off = wp_off.offsets[(size_t) id]; - auto it = off_to_id.find(off); - if (it != off_to_id.end() && it->second != id) { - ++n_dup_pairs; - if (!viol && n_viol < 8) { - fprintf(stderr, "[vk-wp-idchk] COLLISION src0=%s: experts %d and %d " - "both map to pool offset %u blocks\n", - ggml_get_name(src0), it->second, id, off); - } - viol = true; - } else { - off_to_id[off] = id; - } - } - if (viol) { ++n_viol; } +static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) { + switch (op) { + case GGML_OP_GET_ROWS: + GGML_ASSERT(src1->type == GGML_TYPE_I32); + if (src0->type == GGML_TYPE_I32) { + // i32 src only supports i32 result + GGML_ASSERT(dst->type == GGML_TYPE_I32); + return ctx->device->pipeline_get_rows[src0->type]; } - if (n_disp % 100 == 0) { - fprintf(stderr, "[vk-wp-idchk] dispatches=%zu with_collision=%zu dup_pairs=%zu\n", - n_disp, n_viol, n_dup_pairs); + if (dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_get_rows[src0->type]; } - } - - // GGML_VK_WP_EXPDUMP=<path>: dump the first 32 bytes the shader would read - // for EVERY expert of one named tensor, using exactly the addressing the - // dispatch is about to use. Run once paged and once not, then diff: identical - // means expert e really is at the slot published for expert e; a permutation - // means the expert->slot association is wrong, which every other check so far - // is blind to because they are all keyed by offset rather than identity. - { - static const char * dump_path = getenv("GGML_VK_WP_EXPDUMP"); - static bool dumped = false; - const char * nm = ggml_get_name(src0); - if (dump_path != nullptr && !dumped && nm != nullptr && - strcmp(nm, "blk.2.ffn_gate_exps.weight") == 0) { - dumped = true; - const size_t blk = ggml_type_size(src0->type); - const uint32_t bpe = (uint32_t) ((ne00 * ne01) / ggml_blck_size(src0->type)); - FILE * f = fopen(dump_path, "w"); - if (f != nullptr) { - fprintf(f, "# tensor=%s type=%s paged=%d blk=%zu blocks_per_expert=%u\n", - nm, ggml_type_name(src0->type), (int) wp_paged, blk, bpe); - const uint32_t n_exp = (uint32_t) src0->ne[2]; - std::vector<uint8_t> b(32); - for (uint32_t e = 0; e < n_exp; ++e) { - const uint32_t off_blocks = (wp_paged && e < wp_off.offsets.size()) - ? wp_off.offsets[e] : e * bpe; - const size_t abs = (size_t) d_X.offset + (size_t) off_blocks * blk; - if (abs + b.size() > d_X.buffer->size) { continue; } - ggml_vk_buffer_read(d_X.buffer, abs, b.data(), b.size()); - fprintf(f, "expert %2u off_blocks %10u ", e, off_blocks); - for (uint8_t v : b) { fprintf(f, "%02x", v); } - fprintf(f, "\n"); + if (dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_get_rows_f32[src0->type]; + } + return nullptr; + case GGML_OP_GET_ROWS_BACK: + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_get_rows_back_f32; + } + return nullptr; + case GGML_OP_ACC: + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_acc_f32; + } + return nullptr; + case GGML_OP_SET: + if (src0->type == src1->type && src0->type == dst->type && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32)) { + return ctx->device->pipeline_set_f32; + } + return nullptr; + case GGML_OP_ADD: + case GGML_OP_SUB: + case GGML_OP_MUL: + case GGML_OP_DIV: + if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || + (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) || + (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16)) { + return nullptr; + } + switch (op) { + case GGML_OP_ADD: + { + if (ctx->num_additional_fused_ops > 0) { + if (ctx->do_add_rms_partials) { + return ctx->device->pipeline_multi_add_rms[ctx->num_additional_fused_ops]; + } else { + return ctx->device->pipeline_multi_add[ctx->num_additional_fused_ops]; } - // The experts the SHADER will actually use, straight out of the - // ids tensor at dispatch time. If this set differs from the set - // the pager staged, every selected-but-unstaged expert silently - // reads the first_active_slot fallback. - const size_t n_ids = (size_t) nei0 * (size_t) nei1; - std::vector<int32_t> ids_host(n_ids); - ggml_vk_buffer_read(d_ids.buffer, d_ids.offset, - ids_host.data(), n_ids * sizeof(int32_t)); - fprintf(f, "# ids tensor (nei0=%u nei1=%u):", (uint32_t) nei0, (uint32_t) nei1); - for (int32_t v : ids_host) { fprintf(f, " %d", v); } - fprintf(f, "\n"); - fclose(f); - fprintf(stderr, "[vk-wp-expdump] wrote %s (paged=%d)\n", dump_path, (int) wp_paged); + } + if (ctx->do_add_rms_partials) { + auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_add_rms_norepeat : ctx->device->pipeline_add_rms; + return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; + } else { + auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_add_norepeat : ctx->device->pipeline_add; + return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; } } - } - - // Loop over the batch dimension - for (uint32_t expert_i1 = 0; expert_i1 < nei1; ++expert_i1) { - const vk_mat_vec_id_push_constants pc = { - (uint32_t)ne00, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne01, - stride_batch_x, stride_batch_y, (uint32_t)(ne20 * ne21), - fusion_flags, - (uint32_t)nei0, (uint32_t)ne11, expert_i1, nbi1, - wp_paged ? 1u : 0u - }; - ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, - { - d_X, - d_Y, - d_D, - d_F0, - d_F1, - d_ids, - d_WP, - }, - pc, { groups_x, (uint32_t)nei0, groups_z }); - } - - if (x_non_contig) { - ctx->prealloc_x_need_sync = true; - } - if (y_non_contig || quantize_y) { - ctx->prealloc_y_need_sync = true; - } -} - -static bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx) { - ggml_tensor * dst = cgraph->nodes[node_idx]; - ggml_tensor * src0 = dst->src[0]; - ggml_tensor * src1 = dst->src[1]; - ggml_tensor * src2 = dst->src[2]; - if (ggml_vk_mul_mat_id_force_mm(src1->ne[2])) { - return false; - } - // A pinned op normally takes the matmul-id path (that is what the hint was - // introduced for). GGML_VK_MUL_MAT_ID_PIN_KERNEL=vec opts the *expanded - // prefill* shape into the mat-vec-id path instead: that shader already - // addresses src1 as (route, token) -- b_offset = (expert_i0 % ne11)*stride_b - // + expert_i1*batch_stride_b in mul_mat_vec_base.glsl -- and already honours - // an arena-strided src0 through stride_batch_x, so no copy and no shader - // change is needed. The <= 8 token cap below is a batch-size heuristic for - // the unpinned decode path and does not apply here. - bool pinned_vec = false; - if (ggml_vk_mul_mat_id_hint_pinned(dst)) { - if (ggml_vk_mul_mat_id_pin_kernel() != VK_MUL_MAT_ID_PIN_KERNEL_VEC || - !ggml_vk_mul_mat_id_expanded_rows(dst)) { - return false; + case GGML_OP_SUB: + { + auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_sub_norepeat : ctx->device->pipeline_sub; + return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; } - pinned_vec = true; - } - const uint64_t type_size = ggml_type_size(src0->type); - const uint64_t block_size = ggml_blck_size(src0->type); - const uint64_t canonical_nb2 = - (src0->ne[0] * src0->ne[1] / block_size) * type_size; - const bool strided = src0->nb[2] != canonical_nb2; - const bool stride_representable = type_size != 0 && block_size != 0 && - src0->nb[2] % type_size == 0 && - (src0->nb[2] / type_size) <= UINT32_MAX / block_size; - const bool supported_type = - src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type); - return supported_type && (pinned_vec || src2->ne[1] <= 8) && - (!strided || stride_representable); -} - -static void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { - ggml_tensor * dst = cgraph->nodes[node_idx]; - ggml_tensor * src0 = dst->src[0]; - ggml_tensor * src1 = dst->src[1]; - ggml_tensor * src2 = dst->src[2]; - VK_LOG_DEBUG("ggml_vk_mul_mat_id(" << src0 << ", " << src1 << ", " << src2 << ", " << dst << ")"); - const bool use_vec = ggml_vk_use_mul_mat_vec_id(cgraph, node_idx); - // GGML_VK_WP_FORK=1: which mul_mat_id implementation is actually running. - // The vec path serves batches of <= 8 tokens (plus a pinned expanded-prefill - // op under GGML_VK_MUL_MAT_ID_PIN_KERNEL=vec). MM handles wider batches and - // strided arena weights whose stride cannot be represented by batch_stride_a. - static const bool wp_fork = getenv("GGML_VK_WP_FORK") != nullptr; - if (wp_fork) { - static size_t n_vec = 0, n_mm = 0; - (use_vec ? n_vec : n_mm)++; - if (!use_vec && n_mm <= 4) { - fprintf(stderr, "[vk-wp-fork] mm path src0=%s tokens=%d\n", - ggml_get_name(src0), (int) src2->ne[1]); + case GGML_OP_MUL: + { + auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_mul_norepeat : ctx->device->pipeline_mul; + return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; } - if ((n_vec + n_mm) % 100 == 0) { - fprintf(stderr, "[vk-wp-fork] mul_mat_id vec=%zu mm=%zu\n", n_vec, n_mm); + case GGML_OP_DIV: + { + auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_div_norepeat : ctx->device->pipeline_div; + return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; + } + default: + break; + } + return nullptr; + case GGML_OP_ADD_ID: + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_add_id_f32; + } + return nullptr; + case GGML_OP_OUT_PROD: + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_out_prod_f32; + } + return nullptr; + case GGML_OP_CONCAT: { + if (!ggml_vk_concat_supported(src0, src1, dst)) { + return nullptr; + } + switch (ggml_vk_concat_unit_size(src0->type)) { + case 1: + return ctx->device->pipeline_concat_i8; + case 2: + return ctx->device->pipeline_concat_i16; + case 4: + return ctx->device->pipeline_concat_i32; + case 8: + return ctx->device->pipeline_concat_i64; + default: + return nullptr; } } - if (use_vec) { - ggml_vk_mul_mat_vec_id_q_f16(ctx, subctx, cgraph, node_idx); - } else { - ggml_vk_mul_mat_id_q_f16(ctx, subctx, src0, src1, src2, dst); - } -} - -static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) { - GGML_UNUSED(f32acc); - // Needs to be kept up to date on shader changes - const uint32_t wg_size = params.workgroup_size; - const uint32_t Br = params.block_rows; - const uint32_t Bc = params.block_cols; - - // BF16 uses the fp32 shader (FLOAT_TYPE=float) - const uint32_t float_type_size = (device->fp16 && k_type != GGML_TYPE_BF16) ? sizeof(ggml_fp16_t) : sizeof(float); - - const bool mmq = ggml_vk_fa_scalar_uses_mmq(device, k_type, v_type); - - // tmpsh is overestimated slightly - const uint32_t tmpsh = wg_size * sizeof(float); - const uint32_t tmpshv4 = wg_size * 4 * float_type_size; - - const uint32_t masksh = Bc * (Br + 1) * float_type_size; - // DATA_A_IQ4_NL is compiled into the FA shaders unconditionally, so its shared table is always allocated. - const uint32_t iq_shmem = 16 * float_type_size; - - uint32_t Qf, kvsh, kblocksh_size; - if (mmq) { - // block_b_cache: int32_t qs[8] + FLOAT_TYPEV2 ds - const uint32_t block_b_size = 8 * sizeof(int32_t) + 2 * float_type_size; - Qf = Br * (hsk / 32) * block_b_size; - - // kvsh uses D = HSV (K goes through kblocksh instead) - kvsh = params.shmem_staging ? Bc * (hsv / 4 + 1) * 4 * float_type_size : 4 * float_type_size; - - // The mixed MMQ shader uses a superset block_a_cache that fits every - // FA-supported quant: int32_t qs[8] + uint32_t qh + FLOAT_TYPEV2 dm. - // Single-scale types leave dm.y unused; non-Q5_* leave qh unused. - const uint32_t block_a_size = 8 * sizeof(int32_t) + sizeof(uint32_t) + 2 * float_type_size; - kblocksh_size = params.shmem_staging ? Bc * (hsk / 32) * block_a_size : block_a_size; - } else { - Qf = Br * (hsk / 4 + 1) * 4 * float_type_size; - - const uint32_t D = std::max(hsk, hsv); - kvsh = params.shmem_staging ? Bc * (D / 4 + 1) * 4 * float_type_size : 4 * float_type_size; - - kblocksh_size = 0; - } - - const uint32_t total_size = tmpsh + tmpshv4 + masksh + iq_shmem + Qf + kvsh + kblocksh_size; - const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; - - VK_LOG_DEBUG("ggml_vk_flash_attn_scalar_shmem_support(HSK=" << hsk << ", HSV=" << hsv << ", mmq=" << mmq << ", total_size=" << total_size << ", supported=" << supported); - - return supported; -} - -static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) { - GGML_UNUSED(v_type); - // Needs to be kept up to date on shader changes - const uint32_t Br = params.block_rows; - const uint32_t Bc = params.block_cols; - - const uint32_t MatBr = 16, MatBc = 16; - - const uint32_t row_split = Bc / MatBc; - - const uint32_t hsk_pad = ROUNDUP_POW2(hsk, 16); - const uint32_t hsv_pad = ROUNDUP_POW2(hsv, 16); - - const uint32_t acctype = f32acc ? 4 : 2; - const uint32_t f16vec4 = 8; - - const uint32_t tmpsh = (Bc / MatBc) * sizeof(float); - // DATA_A_IQ4_NL is compiled into the FA shaders unconditionally, so its shared table is always allocated. - const uint32_t iq_shmem = 16 * sizeof(ggml_fp16_t); - - const uint32_t qstride = hsk_pad / 4 + 2; - const uint32_t Qf = Br * qstride * f16vec4; - - const uint32_t psh_stride = Br / 4 + 2; - const uint32_t Psh = Bc * psh_stride * f16vec4; - - const uint32_t sfshstride = (hsk <= 128) ? (Br + 8) : Br; - const uint32_t sfsh = Bc * sfshstride * acctype; - - const uint32_t kvshstride = (params.shmem_staging ? std::max(hsk_pad, hsv_pad) : MatBr) / 4 + 2; - const uint32_t vsh_stride = MatBc / 4 * row_split; - const uint32_t ksh = ((kvshstride >= vsh_stride) ? (Bc * kvshstride) : (Bc * vsh_stride)) * f16vec4; - - // BF16 PVMat accumulator is f32 (no bf16 accumulator support), so pvsh is vec4 (16 bytes) - const uint32_t pvsh_elem_size = (k_type == GGML_TYPE_BF16) ? 16u : f16vec4; - const uint32_t osh_stride = params.row_split * MatBr / 4; - const uint32_t pvsh = MatBc * osh_stride * pvsh_elem_size; - - const uint32_t slope = Br * acctype; - - const uint32_t total_size = tmpsh + iq_shmem + Qf + Psh + sfsh + ksh + pvsh + slope; - const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; - - VK_LOG_DEBUG("ggml_vk_flash_attn_coopmat_shmem_support(HSK=" << hsk << ", HSV=" << hsv << ", f32acc=" << f32acc << ", total_size=" << total_size << ", supported=" << supported); - - return supported; -} - -// ---- outlier-matrix sweep (2026-07-01): runtime centroid-table toggle ---- -// -// Vulkan-side equivalent of ggml_cuda_turbo4_64_ol_use_n64_table() in -// turbo-quant.cuh. Temporary, investigative-only mechanism (see task brief -// .superpowers/sdd/outlier-matrix-task-brief.md): GGML_TURBO4_64_OL_TABLE=n64 -// selects the N=64-recalibrated centroid table for the non-outlier elements -// of turbo4_64_ol/ol8/ol12; anything else (including unset) keeps the -// default table. Read once on the host per process and cached, then passed -// down as the use_n64_table push-constant field to the scatter/attn/decode -// pipelines whenever the KV cache type is one of the turbo4_64_ol family. -static bool ggml_vk_turbo4_64_ol_use_n64_table() { - static const bool use_n64 = []() { - const char * v = getenv("GGML_TURBO4_64_OL_TABLE"); - return v != nullptr && strcmp(v, "n64") == 0; - }(); - return use_n64; -} - -// SP2: paged attention (GGML_OP_PAGED_ATTN_MT). Mirrors the CUDA dispatch -// (mt::ggml_cuda_op_paged_attn_mt): scatter K_cur/V_cur into the paged cache, -// barrier, then prefill attention. F16 cache path (Task 3); Task 4 adds turbo4_0. -static void ggml_vk_paged_attn_mt(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - const ggml_tensor * q = dst->src[0]; - const ggml_tensor * k_cache = dst->src[1]; - const ggml_tensor * v_cache = dst->src[2]; - const ggml_tensor * block_tables = dst->src[3]; - const ggml_tensor * context_lens = dst->src[4]; - const ggml_tensor * q_lens = dst->src[5]; - const ggml_tensor * k_cur = dst->src[6]; - const ggml_tensor * v_cur = dst->src[7]; - const ggml_tensor * slot_mapping = dst->src[8]; - - // op_params: [0]=float scale, [1]=block_size, [2]=max_blocks_per_seq, [3]=n_kv_heads. - const float * op_params_f = (const float *)dst->op_params; - const float scale = op_params_f[0]; - const uint32_t block_size = ((const int32_t *)dst->op_params)[1]; - const uint32_t max_bps = ((const int32_t *)dst->op_params)[2]; - const uint32_t n_kv_heads = ((const int32_t *)dst->op_params)[3]; - - const uint32_t head_size = (uint32_t)q->ne[0]; - const uint32_t n_heads = (uint32_t)q->ne[1]; - const uint32_t n_tokens = (uint32_t)q->ne[2]; - const uint32_t n_seq = (uint32_t)block_tables->ne[1]; - - const ggml_type cache_type = k_cache->type; - vk_pipeline scatter_pipeline = ctx->device->pipeline_paged_attn_scatter[cache_type]; - vk_pipeline attn_pipeline = ctx->device->pipeline_paged_attn[cache_type]; - GGML_ASSERT(scatter_pipeline != nullptr && attn_pipeline != nullptr); - - // Decode vs prefill routing. Decode (the split-K single-query fast path) is - // valid only when EVERY sequence has q_len == 1. Each seq always has at - // least one query token, so the sum of q_lens (== n_tokens == dst->ne[2]) - // equals n_seq iff every q_len is exactly 1 — a shape-only test, no host - // read of the q_lens tensor (no GPU stall). Anything else falls back to the - // Task 3-4 prefill path, which is always correct. - const bool decode = (n_tokens == n_seq); - vk_pipeline decode_pipeline = ctx->device->pipeline_paged_attn_decode[cache_type]; - vk_pipeline reduce_pipeline = ctx->device->pipeline_paged_attn_decode_reduce; - - // Partials buffer stride: one chunk per CHUNK_KV (=128) tokens of allocated - // context (max_blocks_per_seq * block_size is an upper bound on real ctx). - const uint32_t CHUNK_KV = 128u; - const uint32_t num_splits = (max_bps * block_size + CHUNK_KV - 1u) / CHUNK_KV; - - ggml_pipeline_request_descriptor_sets(ctx, scatter_pipeline, 1); - if (decode) { - GGML_ASSERT(decode_pipeline != nullptr && reduce_pipeline != nullptr); - ggml_pipeline_request_descriptor_sets(ctx, decode_pipeline, 1); - ggml_pipeline_request_descriptor_sets(ctx, reduce_pipeline, 1); - } else { - ggml_pipeline_request_descriptor_sets(ctx, attn_pipeline, 1); - } - - vk_subbuffer q_buf = ggml_vk_tensor_subbuffer(ctx, q); - vk_subbuffer k_cache_buf = ggml_vk_tensor_subbuffer(ctx, k_cache); - vk_subbuffer v_cache_buf = ggml_vk_tensor_subbuffer(ctx, v_cache); - vk_subbuffer block_tables_buf = ggml_vk_tensor_subbuffer(ctx, block_tables); - vk_subbuffer context_lens_buf = ggml_vk_tensor_subbuffer(ctx, context_lens); - vk_subbuffer q_lens_buf = ggml_vk_tensor_subbuffer(ctx, q_lens); - vk_subbuffer k_cur_buf = ggml_vk_tensor_subbuffer(ctx, k_cur); - vk_subbuffer v_cur_buf = ggml_vk_tensor_subbuffer(ctx, v_cur); - vk_subbuffer slot_mapping_buf = ggml_vk_tensor_subbuffer(ctx, slot_mapping); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); - - // Phase 1: scatter K_cur/V_cur into the paged cache. - // grid = { n_tokens, n_kv_heads * n_qblk, 2 } (z selects K vs V; y fans out - // per 128-element quant-block so head_dim multiples of 128 are supported). - const bool is_t64_ol_family = (cache_type == GGML_TYPE_TURBO4_64_OL || - cache_type == GGML_TYPE_TURBO4_64_OL8 || - cache_type == GGML_TYPE_TURBO4_64_OL12); - const uint32_t qblk_elems = (cache_type == GGML_TYPE_Q8_0) ? 32u : - (cache_type == GGML_TYPE_TURBO4_64 || is_t64_ol_family) ? 64u : 128u; // q8_0: 32-elt blocks, turbo4_64/turbo4_64_ol family: 64-elt blocks - const uint32_t n_qblk = head_size / qblk_elems; // head_dim is a multiple of qblk_elems - // Outlier-matrix sweep (investigative-only, see ggml_vk_turbo4_64_ol_use_n64_table - // above): only meaningful for the turbo4_64_ol/ol8/ol12 family; 0 (default - // table) for every other cache type. - const uint32_t use_n64_table = (is_t64_ol_family && ggml_vk_turbo4_64_ol_use_n64_table()) ? 1u : 0u; - const vk_op_paged_scatter_pc scatter_pc = { head_size, block_size, n_kv_heads, n_tokens, use_n64_table }; - ggml_vk_dispatch_pipeline(ctx, subctx, scatter_pipeline, - { slot_mapping_buf, k_cur_buf, v_cur_buf, k_cache_buf, v_cache_buf }, - scatter_pc, { n_tokens, n_kv_heads * n_qblk, 2 }); - - // Barrier so attention reads the freshly scattered cache. - ggml_vk_sync_buffers(ctx, subctx); - - if (decode) { - // Phase 2 (decode): split-K. Pass-1 computes per-(seq, head, chunk) - // partials into a scratch buffer; pass-2 merges them with a log-sum-exp - // combine. Mirrors the flash-attention split-K scratch mechanism: size - // ctx->prealloc_split_k via prealloc_size_split_k and reuse that buffer. - const uint64_t partials_size = - (uint64_t)num_splits * n_seq * n_heads * (head_size + 2) * sizeof(float); - if (partials_size > ctx->device->properties.limits.maxStorageBufferRange) { - GGML_ABORT("paged decode partials buffer too large"); - } - if (ctx->prealloc_size_split_k < partials_size) { - ctx->prealloc_size_split_k = partials_size; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if (ctx->prealloc_split_k_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - vk_subbuffer partials_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_split_k, 0); - - // Pass 1: per-chunk partials. grid = { n_seq, n_heads, num_splits }. - const vk_op_paged_decode_pc decode_pc = { - head_size, block_size, n_heads, n_kv_heads, max_bps, n_seq, num_splits, scale, use_n64_table - }; - ggml_vk_dispatch_pipeline(ctx, subctx, decode_pipeline, - { q_buf, k_cache_buf, v_cache_buf, block_tables_buf, context_lens_buf, q_lens_buf, partials_buf }, - decode_pc, { n_seq, n_heads, num_splits }); - - // Barrier so the reduce reads completed partials. - ggml_vk_sync_buffers(ctx, subctx); - - // Pass 2: reduce. grid = { n_seq, n_heads, 1 }. - const vk_op_paged_decode_reduce_pc reduce_pc = { head_size, n_heads, n_seq, num_splits }; - ggml_vk_dispatch_pipeline(ctx, subctx, reduce_pipeline, - { partials_buf, context_lens_buf, dst_buf }, - reduce_pc, { n_seq, n_heads, 1 }); - - ctx->prealloc_split_k_need_sync = true; - } else { - // Phase 2: prefill attention. grid = { n_tokens, n_heads, 1 }. - const vk_op_paged_attn_pc attn_pc = { - head_size, block_size, n_heads, n_kv_heads, max_bps, n_seq, n_tokens, scale, use_n64_table - }; - ggml_vk_dispatch_pipeline(ctx, subctx, attn_pipeline, - { q_buf, k_cache_buf, v_cache_buf, block_tables_buf, context_lens_buf, q_lens_buf, dst_buf }, - attn_pc, { n_tokens, n_heads, 1 }); - } -} - -static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v, const ggml_tensor * mask, const ggml_tensor * sinks, ggml_tensor * dst) { - VK_LOG_DEBUG("ggml_vk_flash_attn((" << q << ", name=" << q->name << ", type=" << q->type << ", ne0=" << q->ne[0] << ", ne1=" << q->ne[1] << ", ne2=" << q->ne[2] << ", ne3=" << q->ne[3] << ", nb0=" << q->nb[0] << ", nb1=" << q->nb[1] << ", nb2=" << q->nb[2] << ", nb3=" << q->nb[3]; - std::cerr << "), (" << k << ", name=" << k->name << ", type=" << k->type << ", ne0=" << k->ne[0] << ", ne1=" << k->ne[1] << ", ne2=" << k->ne[2] << ", ne3=" << k->ne[3] << ", nb0=" << k->nb[0] << ", nb1=" << k->nb[1] << ", nb2=" << k->nb[2] << ", nb3=" << k->nb[3]; - std::cerr << "), (" << v << ", name=" << v->name << ", type=" << v->type << ", ne0=" << v->ne[0] << ", ne1=" << v->ne[1] << ", ne2=" << v->ne[2] << ", ne3=" << v->ne[3] << ", nb0=" << v->nb[0] << ", nb1=" << v->nb[1] << ", nb2=" << v->nb[2] << ", nb3=" << v->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; - if (sinks) { - std::cerr << "), (" << sinks << ", name=" << sinks->name << ", type=" << sinks->type << ", ne0=" << sinks->ne[0] << ", ne1=" << sinks->ne[1] << ", ne2=" << sinks->ne[2] << ", ne3=" << sinks->ne[3] << ", nb0=" << sinks->nb[0] << ", nb1=" << sinks->nb[1] << ", nb2=" << sinks->nb[2] << ", nb3=" << sinks->nb[3]; - } - std::cerr << "))"); - - GGML_TENSOR_LOCALS(int64_t, neq, q, ne) - GGML_TENSOR_LOCALS(size_t, nbq, q, nb) - GGML_TENSOR_LOCALS(int64_t, nek, k, ne) - GGML_TENSOR_LOCALS(size_t, nbk, k, nb) - GGML_TENSOR_LOCALS(int64_t, nev, v, ne) - GGML_TENSOR_LOCALS(size_t, nbv, v, nb) - GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) - GGML_TENSOR_LOCALS(size_t, nb, dst, nb) - - const uint32_t nem0 = mask ? mask->ne[0] : 0; - const uint32_t nem1 = mask ? mask->ne[1] : 0; - const uint32_t nem2 = mask ? mask->ne[2] : 0; - const uint32_t nem3 = mask ? mask->ne[3] : 0; - - const uint32_t HSK = nek0; - const uint32_t HSV = nev0; - uint32_t N = neq1; - const uint32_t KV = nek1; - - GGML_ASSERT(ne0 == HSV); - GGML_ASSERT(ne2 == N); - - // input tensor rows must be contiguous - GGML_ASSERT(nbq0 == ggml_type_size(q->type)); - GGML_ASSERT(nbk0 == ggml_type_size(k->type)); - GGML_ASSERT(nbv0 == ggml_type_size(v->type)); - - GGML_ASSERT(neq0 == HSK); - - GGML_ASSERT(neq1 == N); - - GGML_ASSERT(nev1 == nek1); - - // dst cannot be transposed or permuted - GGML_ASSERT(nb0 == sizeof(float)); - GGML_ASSERT(nb0 <= nb1); - GGML_ASSERT(nb1 <= nb2); - GGML_ASSERT(nb2 <= nb3); - - assert(dst->type == GGML_TYPE_F32); - assert(q->type == GGML_TYPE_F32); - uint32_t gqa_ratio = 1; - uint32_t qk_ratio = neq2 / nek2; - uint32_t workgroups_x = (uint32_t)neq1; - uint32_t workgroups_y = (uint32_t)neq2; - uint32_t workgroups_z = (uint32_t)neq3; - - const bool f32acc = !ctx->device->fp16 || dst->op_params[3] == GGML_PREC_F32 || k->type == GGML_TYPE_BF16; - - // dequant K/V once into an f16 scratch, reordered KV layout so FA can read without a stride - auto is_dense_kv_cache = [](const ggml_tensor * t) { - return t->nb[0] == ggml_type_size(t->type) && - t->nb[2] == ggml_row_size(t->type, t->ne[0]) && - t->nb[1] == t->nb[2] * t->ne[2] && - (t->ne[3] == 1 || t->nb[3] == t->nb[1] * t->ne[1]); - }; - const bool k_quant = k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_BF16 && k->type != GGML_TYPE_F32; - const bool v_quant = v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_BF16 && v->type != GGML_TYPE_F32; - const bool use_dequant_kv = k_quant && v_quant && neq1 >= 64 && - is_dense_kv_cache(k) && is_dense_kv_cache(v) && - (uint64_t)ggml_nelements(k) * sizeof(ggml_fp16_t) <= ctx->device->properties.limits.maxStorageBufferRange && - (uint64_t)ggml_nelements(v) * sizeof(ggml_fp16_t) <= ctx->device->properties.limits.maxStorageBufferRange && - ctx->device->pipeline_dequant_transpose[k->type] != nullptr && - ctx->device->pipeline_dequant_transpose[v->type] != nullptr && - // coopmat2 path does not benefit from the f16 scratch - !ctx->device->coopmat2 && - // Intel Xe1 regresses, see PR 25494 - (ctx->device->vendor_id != VK_VENDOR_ID_INTEL || - (ctx->device->coopmat_support && ctx->device->architecture != vk_device_architecture::INTEL_XE1)); - const ggml_type k_type_eff = use_dequant_kv ? GGML_TYPE_F16 : k->type; - const ggml_type v_type_eff = use_dequant_kv ? GGML_TYPE_F16 : v->type; - - // For scalar/coopmat1 FA, we can use the "large" size to accommodate qga. - // For coopmat2 FA, we always use the small size (which is still pretty large for gqa). - vk_fa_tuning_params tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, 512, KV, k_type_eff, v_type_eff, f32acc); - const uint32_t max_gqa = std::min(tuning_params.block_rows, 32u); - - if (N <= 8 && qk_ratio > 1 && qk_ratio <= max_gqa && - qk_ratio * nek2 == neq2 && nek2 == nev2 && nem2 <= 1) { - // grouped query attention - make the N dimension equal to gqa_ratio, reduce - // workgroups proportionally in y dimension. The shader will detect gqa_ratio > 1 - // and change addressing calculations to index Q's dimension 2. - gqa_ratio = qk_ratio; - N = gqa_ratio; - workgroups_y /= gqa_ratio; - } - - tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, N, KV, k_type_eff, v_type_eff, f32acc); - - const uint32_t q_stride = (uint32_t)(nbq1 / ggml_type_size(q->type)); - uint32_t k_stride = (uint32_t)(nbk1 / ggml_type_size(k->type)); - uint32_t v_stride = (uint32_t)(nbv1 / ggml_type_size(v->type)); - - // For F32, the shader treats it as a block of size 4 (for vec4 loads) - if (k->type == GGML_TYPE_F32) { - k_stride /= 4; - } - if (v->type == GGML_TYPE_F32) { - v_stride /= 4; - } - - uint32_t nbk2_eff = (uint32_t)nbk2, nbk3_eff = (uint32_t)nbk3; - uint32_t nbv2_eff = (uint32_t)nbv2, nbv3_eff = (uint32_t)nbv3; - if (use_dequant_kv) { - k_stride = HSK; - v_stride = HSV; - nbk2_eff = (uint32_t)((uint64_t)HSK * KV * sizeof(ggml_fp16_t)); - nbk3_eff = (uint32_t)((uint64_t)HSK * KV * nek2 * sizeof(ggml_fp16_t)); - nbv2_eff = (uint32_t)((uint64_t)HSV * KV * sizeof(ggml_fp16_t)); - nbv3_eff = (uint32_t)((uint64_t)HSV * KV * nev2 * sizeof(ggml_fp16_t)); - } - - const uint32_t alignment = tuning_params.block_cols; - bool aligned = (KV % alignment) == 0 && - // the "aligned" shader variant will forcibly align strides, for performance - (q_stride & 7) == 0 && (k_stride & 7) == 0 && (v_stride & 7) == 0; - - // Need to use the coopmat2 variant that clamps loads when HSK/HSV aren't sufficiently aligned. - if (((HSK | HSV) % 16) != 0 && tuning_params.path == FA_COOPMAT2) { - aligned = false; - } - - float scale = 1.0f; - float max_bias = 0.0f; - float logit_softcap = 0.0f; - - memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); - memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); - memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); - - if (logit_softcap != 0) { - scale /= logit_softcap; - } - - // Only use mask opt when the mask is fairly large. This hasn't been tuned extensively. - bool use_mask_opt = mask && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16 - && (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK > 256 || HSV > 256); - vk_fa_pipeline_state fa_pipeline_state = get_fa_pipeline_state(ctx->device, tuning_params, HSK, HSV, aligned, f32acc, - mask != nullptr, use_mask_opt, logit_softcap != 0, k_type_eff, v_type_eff); - - vk_pipeline pipeline = nullptr; - - { - std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); - auto &pipelines = ctx->device->pipeline_flash_attn_f32_f16; - auto it = pipelines.find(fa_pipeline_state); - if (it != pipelines.end()) { - pipeline = it->second; - } else { - pipelines[fa_pipeline_state] = pipeline = std::make_shared<vk_pipeline_struct>(); - } - } - - assert(pipeline); - // Compile early to initialize wg_denoms. - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - - uint32_t split_kv = KV; - uint32_t split_k = 1; - - // Intel Alchemist prefers more workgroups - const uint32_t shader_core_count_multiplier = (ctx->device->vendor_id == VK_VENDOR_ID_INTEL && ctx->device->architecture != INTEL_XE2) ? 2 : 1; - - // Use a placeholder core count if one isn't available. split_k is a big help for perf. - const uint32_t shader_core_count = ctx->device->shader_core_count ? ctx->device->shader_core_count * shader_core_count_multiplier : 16; - - const uint32_t Br = fa_pipeline_state.Br; - const uint32_t Bc = fa_pipeline_state.Bc; - - GGML_ASSERT(Br == pipeline->wg_denoms[0]); - const uint32_t Tr = CEIL_DIV(N, Br); - - // Try to use split_k when KV is large enough to be worth the overhead. - if (gqa_ratio > 1 && workgroups_x <= Br) { - split_k = shader_core_count * 2 / (workgroups_x * workgroups_y * workgroups_z); - } else if (gqa_ratio <= 1) { - uint32_t total_wgs_no_split = Tr * workgroups_y * workgroups_z; - if (total_wgs_no_split < shader_core_count * 2) { - split_k = shader_core_count * 2 / total_wgs_no_split; - } - } - - if (split_k > 1) { - // Try to evenly split KV into split_k chunks, but it needs to be a multiple - // of "align", so recompute split_k based on that. - split_kv = ROUNDUP_POW2(std::max(1u, KV / split_k), alignment); - split_k = CEIL_DIV(KV, split_kv); - } - - // Reserve space for split_k temporaries. For each split x batch, we need to store the O matrix (D x ne1) - // and the per-row m and L values (ne1 rows). We store all the matrices first, followed by the rows. - // For matrices, the order is (inner to outer) [HSV, ne1, k, ne2, ne3]. - // For L/M, the order is (inner to outer) [ne1, k, ne2, ne3]. - const uint64_t split_k_size = split_k > 1 ? (HSV * ne1 * sizeof(float) + ne1 * sizeof(float) * 2) * split_k * ne2 * ne3 : 0; - if (split_k_size > ctx->device->properties.limits.maxStorageBufferRange) { - GGML_ABORT("Requested preallocation size is too large"); - } - if (ctx->prealloc_size_split_k < split_k_size) { - ctx->prealloc_size_split_k = split_k_size; - ggml_vk_preallocate_buffers(ctx, subctx); - } - - const uint32_t mask_opt_num_dwords = CEIL_DIV(nem0, 16 * Bc); - const uint64_t mask_opt_size = sizeof(uint32_t) * mask_opt_num_dwords * CEIL_DIV(nem1, Br) * nem2 * nem3; - - vk_pipeline pipeline_fa_mask_opt = nullptr; - if (use_mask_opt) { - { - std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); - auto &pipelines = ctx->device->pipeline_fa_mask_opt; - auto it = pipelines.find({Br, Bc}); - if (it != pipelines.end()) { - pipeline_fa_mask_opt = it->second; - } else { - pipelines[{Br, Bc}] = pipeline_fa_mask_opt = std::make_shared<vk_pipeline_struct>(); - } - } - assert(pipeline_fa_mask_opt); - ggml_pipeline_request_descriptor_sets(ctx, pipeline_fa_mask_opt, 1); - - if (ctx->prealloc_size_y < mask_opt_size) { - ctx->prealloc_size_y = mask_opt_size; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - } - - const uint32_t n_head_kv = neq2; - const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv)); - const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - - vk_subbuffer q_buf = ggml_vk_tensor_subbuffer(ctx, q); - vk_subbuffer k_buf = ggml_vk_tensor_subbuffer(ctx, k); - vk_subbuffer v_buf = ggml_vk_tensor_subbuffer(ctx, v); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); - vk_subbuffer mask_buf = mask ? ggml_vk_tensor_subbuffer(ctx, mask) : q_buf; - vk_subbuffer sinks_buf = sinks ? ggml_vk_tensor_subbuffer(ctx, sinks) : q_buf; - vk_subbuffer mask_opt_buf = use_mask_opt ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf; - - if (use_dequant_kv) { - const uint64_t fp = sizeof(ggml_fp16_t); - const uint64_t k_f16_sz = (uint64_t)ggml_nelements(k) * fp; - const uint64_t v_f16_sz = (uint64_t)ggml_nelements(v) * fp; - if (ctx->prealloc_size_x < k_f16_sz + v_f16_sz) { - ctx->prealloc_size_x = k_f16_sz + v_f16_sz; - ggml_vk_preallocate_buffers(ctx, subctx); - } - vk_pipeline tr_k = ctx->device->pipeline_dequant_transpose[k->type]; - vk_pipeline tr_v = ctx->device->pipeline_dequant_transpose[v->type]; - ggml_pipeline_request_descriptor_sets(ctx, tr_k, 1); - ggml_pipeline_request_descriptor_sets(ctx, tr_v, 1); - if (ctx->prealloc_x_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - vk_subbuffer k_dst = vk_subbuffer{ ctx->prealloc_x, 0, k_f16_sz }; - vk_subbuffer v_dst = vk_subbuffer{ ctx->prealloc_x, k_f16_sz, v_f16_sz }; - const uint32_t k_nel = (uint32_t)ggml_nelements(k); - const uint32_t v_nel = (uint32_t)ggml_nelements(v); - { const std::vector<uint32_t> pc = { (uint32_t)HSK, (uint32_t)nek2, (uint32_t)KV, 0, k_nel }; - ggml_vk_dispatch_pipeline(ctx, subctx, tr_k, { k_buf, k_dst }, pc, { k_nel, 1, 1 }); } - { const std::vector<uint32_t> pc = { (uint32_t)HSV, (uint32_t)nev2, (uint32_t)KV, 0, v_nel }; - ggml_vk_dispatch_pipeline(ctx, subctx, tr_v, { v_buf, v_dst }, pc, { v_nel, 1, 1 }); } - ggml_vk_sync_buffers(ctx, subctx); - k_buf = k_dst; - v_buf = v_dst; - } - - uint32_t mask_n_head_log2 = ((sinks != nullptr) << 24) | n_head_log2; - - if (use_mask_opt) - { - const vk_op_flash_attn_mask_opt_push_constants opt_pc = { - nem0, - nem1, - nem2, - (uint32_t)(mask->nb[1] / sizeof(ggml_fp16_t)), - (uint32_t)(mask->nb[2] / sizeof(ggml_fp16_t)), - (uint32_t)(mask->nb[3] / sizeof(ggml_fp16_t)), - mask_opt_num_dwords, - mask_opt_num_dwords * CEIL_DIV(nem1, Br), - mask_opt_num_dwords * CEIL_DIV(nem1, Br) * nem2, - }; - - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline_fa_mask_opt, - { mask_buf, mask_opt_buf }, opt_pc, - { mask_opt_num_dwords, CEIL_DIV(nem1, Br), nem2 * nem3 }); - ggml_vk_sync_buffers(ctx, subctx); - } - - const vk_flash_attn_push_constants pc = { N, KV, - (uint32_t)ne1, (uint32_t)ne2, (uint32_t)ne3, - (uint32_t)neq2, (uint32_t)neq3, - (uint32_t)nek2, (uint32_t)nek3, - (uint32_t)nev2, (uint32_t)nev3, - nem1, nem2, nem3, - q_stride, (uint32_t)nbq2, (uint32_t)nbq3, - k_stride, nbk2_eff, nbk3_eff, - v_stride, nbv2_eff, nbv3_eff, - scale, max_bias, logit_softcap, - mask_n_head_log2, m0, m1, - gqa_ratio, split_kv, split_k }; - - if (split_k > 1) { - ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_flash_attn_split_k_reduce, 1); - - if (ctx->prealloc_split_k_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - - // We reuse workgroups_x to mean the number of splits, so we need to - // cancel out the divide by wg_denoms[0]. - uint32_t dispatch_x; - if (gqa_ratio > 1) { - workgroups_x *= pipeline->wg_denoms[0]; - dispatch_x = split_k * workgroups_x; - } else { - dispatch_x = Tr * split_k * pipeline->wg_denoms[0]; - } - - vk_subbuffer split_k_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_split_k, 0); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {q_buf, k_buf, v_buf, mask_buf, sinks_buf, split_k_buf, mask_opt_buf}, - pc, { dispatch_x, workgroups_y, workgroups_z }); - - ggml_vk_sync_buffers(ctx, subctx); - const vk_op_flash_attn_split_k_reduce_push_constants pc2 = { HSV, (uint32_t)ne1, (uint32_t)ne2, (uint32_t)ne3, split_k, (sinks != nullptr) }; - ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_flash_attn_split_k_reduce, - {split_k_buf, sinks_buf, dst_buf}, - pc2, { (uint32_t)ne1, HSV, (uint32_t)(ne2 * ne3) }); - ctx->prealloc_split_k_need_sync = true; - } else { - if (gqa_ratio > 1) { - // When using gqa, we want one actual workgroup per batch, so cancel out wg_denoms - workgroups_x *= pipeline->wg_denoms[0]; - } - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {q_buf, k_buf, v_buf, mask_buf, sinks_buf, dst_buf, mask_opt_buf}, - pc, { workgroups_x, workgroups_y, workgroups_z }); - } - - if (use_dequant_kv) { - ctx->prealloc_x_need_sync = true; - } -} - -static vk_conv_shapes ggml_vk_conv_select_shape(ggml_backend_vk_context * ctx, uint32_t K, uint32_t NPQ) { - auto n_tiles = [&](vk_conv_shapes s) { - return CEIL_DIV(K, vk_conv_block_sizes[s].K) - * CEIL_DIV(NPQ, vk_conv_block_sizes[s].NPQ); - }; - - // We can't query number of shader cores on Intel, use 32 as a placeholder - // so small convolutions will still choose a smaller tile. - const uint32_t shader_core_count = ctx->device->shader_core_count > 0 ? ctx->device->shader_core_count : 32; - - // 128x128 isn't used with cm1 due to shared memory size; fall through to a smaller tile. - bool allow_128x128 = true; -#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - if (!ctx->device->coopmat2 && ctx->device->coopmat_support && ctx->device->coopmat_support_16x16x16_f16acc) { - allow_128x128 = false; - } -#endif - - if (allow_128x128 && K > 64 && n_tiles(CONV_SHAPE_128x128) >= shader_core_count * 2) { - return CONV_SHAPE_128x128; - } else if (K <= 32 && n_tiles(CONV_SHAPE_32x256) >= shader_core_count * 2) { - return CONV_SHAPE_32x256; - } else if (K <= 64 && n_tiles(CONV_SHAPE_64x128) >= shader_core_count * 2) { - return CONV_SHAPE_64x128; - } else if (!allow_128x128 && K > 64 && n_tiles(CONV_SHAPE_64x128) >= shader_core_count * 2) { - // cm1 fallback for large K when 128x128 isn't available - return CONV_SHAPE_64x128; - } else { - return CONV_SHAPE_64x32; - } -} - -static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) { - switch (op) { - case GGML_OP_GET_ROWS: - GGML_ASSERT(src1->type == GGML_TYPE_I32); - if (src0->type == GGML_TYPE_I32) { - // i32 src only supports i32 result - GGML_ASSERT(dst->type == GGML_TYPE_I32); - return ctx->device->pipeline_get_rows[src0->type]; - } - if (dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_get_rows[src0->type]; - } - if (dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_get_rows_f32[src0->type]; - } - return nullptr; - case GGML_OP_GET_ROWS_BACK: - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_get_rows_back_f32; - } - return nullptr; - case GGML_OP_ACC: - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_acc_f32; - } - return nullptr; - case GGML_OP_SET: - if (src0->type == src1->type && src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32)) { - return ctx->device->pipeline_set_f32; - } - return nullptr; - case GGML_OP_ADD: - case GGML_OP_SUB: - case GGML_OP_MUL: - case GGML_OP_DIV: - if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || - (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) || - (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16)) { - return nullptr; - } - switch (op) { - case GGML_OP_ADD: - { - if (ctx->num_additional_fused_ops > 0) { - if (ctx->do_add_rms_partials) { - return ctx->device->pipeline_multi_add_rms[ctx->num_additional_fused_ops]; - } else { - return ctx->device->pipeline_multi_add[ctx->num_additional_fused_ops]; - } - } - if (ctx->do_add_rms_partials) { - auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_add_rms_norepeat : ctx->device->pipeline_add_rms; - return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; - } else { - auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_add_norepeat : ctx->device->pipeline_add; - return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; - } - } - case GGML_OP_SUB: - { - auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_sub_norepeat : ctx->device->pipeline_sub; - return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; - } - case GGML_OP_MUL: - { - auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_mul_norepeat : ctx->device->pipeline_mul; - return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; - } - case GGML_OP_DIV: - { - auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_div_norepeat : ctx->device->pipeline_div; - return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; - } - default: - break; - } - return nullptr; - case GGML_OP_ADD_ID: - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_add_id_f32; - } - return nullptr; - case GGML_OP_OUT_PROD: - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_out_prod_f32; - } - return nullptr; - case GGML_OP_CONCAT: { - if (!ggml_vk_concat_supported(src0, src1, dst)) { - return nullptr; - } - switch (ggml_vk_concat_unit_size(src0->type)) { - case 1: - return ctx->device->pipeline_concat_i8; - case 2: - return ctx->device->pipeline_concat_i16; - case 4: - return ctx->device->pipeline_concat_i32; - case 8: - return ctx->device->pipeline_concat_i64; - default: - return nullptr; - } - } - case GGML_OP_UPSCALE: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - uint32_t mode = (ggml_get_op_params_i32(dst, 0) & (0xFF | GGML_SCALE_FLAG_ANTIALIAS)); - switch (mode) { - case GGML_SCALE_MODE_NEAREST: - return ctx->device->pipeline_upscale_nearest_f32; - case GGML_SCALE_MODE_BILINEAR: - return ctx->device->pipeline_upscale_bilinear_f32; - case GGML_SCALE_MODE_BICUBIC: - return ctx->device->pipeline_upscale_bicubic_f32; - case GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS: - return ctx->device->pipeline_upscale_bilinear_antialias_f32; - default: - return nullptr; - } - } - return nullptr; - case GGML_OP_SCALE: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_scale_f32; - } - return nullptr; - case GGML_OP_SQR: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_sqr[dst->type == GGML_TYPE_F16]; - } - return nullptr; - case GGML_OP_SQRT: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_sqrt[dst->type == GGML_TYPE_F16]; - } - return nullptr; - case GGML_OP_SIN: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_sin[dst->type == GGML_TYPE_F16]; - } - return nullptr; - case GGML_OP_COS: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_cos[dst->type == GGML_TYPE_F16]; - } - return nullptr; - case GGML_OP_LOG: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_log[dst->type == GGML_TYPE_F16]; - } - return nullptr; - case GGML_OP_TRI: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_tri[dst->type == GGML_TYPE_F16]; - } - return nullptr; - case GGML_OP_DIAG: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_diag[dst->type == GGML_TYPE_F16]; - } - return nullptr; - case GGML_OP_CLAMP: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_clamp[dst->type == GGML_TYPE_F16]; - } - return nullptr; - case GGML_OP_PAD: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_pad_f32; - } - return nullptr; - case GGML_OP_PAD_REFLECT_1D: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_pad_reflect_1d_f32; - } - return nullptr; - case GGML_OP_ROLL: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_roll_f32; - } - return nullptr; - case GGML_OP_REPEAT: - if (ggml_type_size(src0->type) == sizeof(float) && ggml_type_size(dst->type) == sizeof(float)) { - return ctx->device->pipeline_repeat_i32; - } - if (ggml_type_size(src0->type) == 2 && ggml_type_size(dst->type) == 2) { - return ctx->device->pipeline_repeat_i16; - } - return nullptr; - case GGML_OP_REPEAT_BACK: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_repeat_back_f32; - } - return nullptr; - case GGML_OP_CPY: - case GGML_OP_CONT: - case GGML_OP_DUP: - return ggml_vk_get_cpy_pipeline(ctx, src0, dst, dst->type); - case GGML_OP_SET_ROWS: - { - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) { - return nullptr; - } - const int src_idx = src0->type == GGML_TYPE_F16; - if (src1->type == GGML_TYPE_I64) { - return ctx->device->pipeline_set_rows_i64[src_idx][dst->type]; - } else if (src1->type == GGML_TYPE_I32) { - return ctx->device->pipeline_set_rows_i32[src_idx][dst->type]; - } - return nullptr; - } - case GGML_OP_SILU_BACK: - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_silu_back_f32; - } - return nullptr; - case GGML_OP_NORM: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_norm_f32; - } - return nullptr; - case GGML_OP_GROUP_NORM: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_group_norm_f32; - } - return nullptr; - case GGML_OP_RMS_NORM: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - if (ctx->do_add_rms_partials) { - return ctx->num_additional_fused_ops > 0 ? ctx->device->pipeline_rms_norm_mul_partials_f32 : ctx->device->pipeline_rms_norm_partials_f32; - } else { - return ctx->num_additional_fused_ops > 0 ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32; - } - } - return nullptr; - case GGML_OP_RMS_NORM_BACK: - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_rms_norm_back_f32; - } - return nullptr; - case GGML_OP_L2_NORM: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_l2_norm_f32; - } - return nullptr; - case GGML_OP_SINKHORN_NORM: - if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return nullptr; - } - switch (src0->ne[0]) { - case 2: return ctx->device->pipeline_sinkhorn_norm_f32[0]; - case 4: return ctx->device->pipeline_sinkhorn_norm_f32[1]; - case 8: return ctx->device->pipeline_sinkhorn_norm_f32[2]; - default: return nullptr; - } - case GGML_OP_UNARY: - if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || - (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) || - (src0->type != dst->type)) { - return nullptr; - } - - switch (ggml_get_unary_op(dst)) { - case GGML_UNARY_OP_EXP: - return ctx->device->pipeline_exp[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_EXPM1: - return ctx->device->pipeline_expm1[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_ELU: - return ctx->device->pipeline_elu[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_SILU: - return ctx->device->pipeline_silu[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_GELU: - return ctx->device->pipeline_gelu[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_GELU_ERF: - return ctx->device->pipeline_gelu_erf[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_GELU_QUICK: - return ctx->device->pipeline_gelu_quick[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_RELU: - return ctx->device->pipeline_relu[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_XIELU: - return ctx->device->pipeline_xielu[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_NEG: - return ctx->device->pipeline_neg[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_TANH: - return ctx->device->pipeline_tanh[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_SIGMOID: - return ctx->device->pipeline_sigmoid[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_HARDSIGMOID: - return ctx->device->pipeline_hardsigmoid[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_HARDSWISH: - return ctx->device->pipeline_hardswish[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_ABS: - return ctx->device->pipeline_abs[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_SOFTPLUS: - return ctx->device->pipeline_softplus[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_STEP: - return ctx->device->pipeline_step[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_ROUND: - return ctx->device->pipeline_round[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_CEIL: - return ctx->device->pipeline_ceil[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_FLOOR: - return ctx->device->pipeline_floor[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_TRUNC: - return ctx->device->pipeline_trunc[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_SGN: - return ctx->device->pipeline_sgn[dst->type == GGML_TYPE_F16]; - default: - break; - } - return nullptr; - case GGML_OP_GLU: - if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || - (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) || - (src0->type != dst->type)) { - return nullptr; - } - - switch (ggml_get_glu_op(dst)) { - case GGML_GLU_OP_GEGLU: - return ctx->device->pipeline_geglu[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_REGLU: - return ctx->device->pipeline_reglu[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_SWIGLU: - return ctx->device->pipeline_swiglu[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_SWIGLU_OAI: - return ctx->device->pipeline_swiglu_oai[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_SWIGLU_CLAMP: - return ctx->device->pipeline_swiglu_clamp[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_GEGLU_ERF: - return ctx->device->pipeline_geglu_erf[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_GEGLU_QUICK: - return ctx->device->pipeline_geglu_quick[dst->type == GGML_TYPE_F16]; - default: - break; - } - return nullptr; - case GGML_OP_DIAG_MASK_INF: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_diag_mask_inf_f32; - } - return nullptr; - case GGML_OP_SOFT_MAX: - GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); - GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F32); - - if (ctx->num_additional_fused_ops) { - uint32_t idx = (uint32_t)ceilf(log2f(float(dst->ne[0]))); - GGML_ASSERT(idx < num_topk_moe_pipelines); - // use n_experts from push constant if it's not equal to the power of two spec constant - bool use_push = dst->ne[0] != (1u << idx); - return ctx->device->pipeline_topk_moe[idx][use_push]; - } - - if (src0->type == GGML_TYPE_F32 && (src1 == nullptr || src1->type == GGML_TYPE_F32) && dst->type == GGML_TYPE_F32) { - return src0->ne[0] > 1024 ? ctx->device->pipeline_soft_max_f32_wg512 : ctx->device->pipeline_soft_max_f32; - } - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) { - return src0->ne[0] > 1024 ? ctx->device->pipeline_soft_max_f32_f16_wg512 : ctx->device->pipeline_soft_max_f32_f16; - } - return nullptr; - case GGML_OP_SOFT_MAX_BACK: - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_soft_max_back_f32; - } - return nullptr; - case GGML_OP_ROPE: - case GGML_OP_ROPE_BACK: - { - const ggml_tensor *rope = ctx->num_additional_fused_ops == 2 ? dst->src[0]->src[0] : dst; - const int mode = ((const int32_t *) rope->op_params)[2]; - const bool is_neox = mode & GGML_ROPE_TYPE_NEOX; - const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; - const bool is_vision = mode == GGML_ROPE_TYPE_VISION; - - if (is_neox) { - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_rope_neox_f32; - } - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_rope_neox_f32_f16; - } - if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_rope_neox_f16; - } - } else if (is_mrope && !is_vision) { - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_rope_multi_f32; - } - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_rope_multi_f32_f16; - } - if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_rope_multi_f16; - } - } else if (is_vision) { - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_rope_vision_f32; - } - if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_rope_vision_f16; - } - } else { - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_rope_norm_f32; - } - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_rope_norm_f32_f16; - } - if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_rope_norm_f16; - } - } - return nullptr; - } - case GGML_OP_SUM: - case GGML_OP_SUM_ROWS: - case GGML_OP_MEAN: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_sum_rows_f32; - } - return nullptr; - case GGML_OP_CROSS_ENTROPY_LOSS: - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return src0->ne[0] > 1024 ? ctx->device->pipeline_cross_entropy_loss_f32_wg512 : ctx->device->pipeline_cross_entropy_loss_f32; - } - return nullptr; - case GGML_OP_CROSS_ENTROPY_LOSS_BACK: - // src0 is the scalar grad; src1 is logits - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && src2 && src2->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return src1->ne[0] > 1024 ? ctx->device->pipeline_cross_entropy_loss_back_f32_wg512 : ctx->device->pipeline_cross_entropy_loss_back_f32; - } - return nullptr; - case GGML_OP_CUMSUM: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - if (src0->ne[0] <= 512) { - return ctx->device->pipeline_cumsum_small_f32; - } else { - return ctx->device->pipeline_cumsum_f32; - } - } - return nullptr; - case GGML_OP_SOLVE_TRI: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - - vk_solve_tri_pipeline_state solve_tri_pipeline_state(src0->ne[0], src1->ne[0]); - - vk_pipeline pipeline = nullptr; - - { - std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); - auto it = ctx->device->pipeline_solve_tri_f32.find(solve_tri_pipeline_state); - if (it != ctx->device->pipeline_solve_tri_f32.end()) { - pipeline = it->second; - } else { - ctx->device->pipeline_solve_tri_f32[solve_tri_pipeline_state] = pipeline = std::make_shared<vk_pipeline_struct>(); - } - } - - return pipeline; - } - return nullptr; - case GGML_OP_ARGMAX: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) { - return ctx->device->pipeline_argmax_f32; - } - return nullptr; - case GGML_OP_COUNT_EQUAL: - if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_I64) { - return ctx->device->pipeline_count_equal_i32; - } - return nullptr; - case GGML_OP_IM2COL: - if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_im2col_f32; - } - if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_im2col_f32_f16; - } - return nullptr; - case GGML_OP_IM2COL_3D: - if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_im2col_3d_f32; - } - if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_im2col_3d_f32_f16; - } - return nullptr; - case GGML_OP_TIMESTEP_EMBEDDING: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_timestep_embedding_f32; - } - return nullptr; - case GGML_OP_CONV_TRANSPOSE_1D: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_conv_transpose_1d_f32; - } - return nullptr; - case GGML_OP_COL2IM_1D: - switch (src0->type) { - case GGML_TYPE_F32: return ctx->device->pipeline_col2im_1d_f32; - case GGML_TYPE_F16: return ctx->device->pipeline_col2im_1d_f16; - case GGML_TYPE_BF16: return ctx->device->pipeline_col2im_1d_bf16; - default: return nullptr; - } - case GGML_OP_POOL_1D: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_pool1d_f32; - } - return nullptr; - case GGML_OP_POOL_2D: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_pool2d_f32; - } - return nullptr; - case GGML_OP_RWKV_WKV6: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_rwkv_wkv6_f32; - } - return nullptr; - case GGML_OP_RWKV_WKV7: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_rwkv_wkv7_f32; - } - return nullptr; - case GGML_OP_GATED_LINEAR_ATTN: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_gated_linear_attn_f32; - } - return nullptr; - case GGML_OP_LIGHTNING_INDEXER: - // only the k type selects a pipeline, the other types are fixed by ggml_lightning_indexer() - if (ggml_vk_lightning_indexer_k_type_supported(src1->type)) { - return ctx->device->pipeline_lightning_indexer_f32[src1->type]; - } - return nullptr; - case GGML_OP_GATED_DELTA_NET: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - const uint32_t S_v = dst->src[2]->ne[0]; - const uint32_t kda = (dst->src[3]->ne[0] == (int64_t)S_v) ? 1 : 0; - uint32_t si; - switch (S_v) { - case 16: si = 0; break; - case 32: si = 1; break; - case 64: si = 2; break; - case 128: si = 3; break; - default: return nullptr; - } - return ctx->device->pipeline_gated_delta_net[si][kda]; - } - return nullptr; - case GGML_OP_SSM_SCAN: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - const uint32_t d_state = src0->ne[0]; - if (d_state == 128) { - return ctx->device->pipeline_ssm_scan_f32_d128; - } else if (d_state == 256) { - return ctx->device->pipeline_ssm_scan_f32_d256; - } - } - return nullptr; - case GGML_OP_SSM_CONV: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - switch (ctx->num_additional_fused_ops) { - case 0: return ctx->device->pipeline_ssm_conv_f32; - case 1: return ctx->device->pipeline_ssm_conv_silu_f32; - case 2: return ctx->device->pipeline_ssm_conv_bias_silu_f32; - default: return nullptr; - } - } - return nullptr; - case GGML_OP_OPT_STEP_ADAMW: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_opt_step_adamw_f32; - } - return nullptr; - case GGML_OP_OPT_STEP_SGD: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_opt_step_sgd_f32; - } - return nullptr; - case GGML_OP_LEAKY_RELU: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_leaky_relu[dst->type == GGML_TYPE_F16]; - } - return nullptr; - case GGML_OP_CONV_2D: - case GGML_OP_CONV_TRANSPOSE_2D: - if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - uint32_t K = dst->ne[2]; // Cout - uint32_t NPQ = dst->ne[3] * dst->ne[1] * dst->ne[0]; // N * OH * OW - vk_conv_shapes shape = ggml_vk_conv_select_shape(ctx, K, NPQ); - - bool transpose = dst->op == GGML_OP_CONV_TRANSPOSE_2D; - uint32_t KW = (uint32_t)src0->ne[0]; - uint32_t KH = (uint32_t)src0->ne[1]; - uint32_t s0 = (uint32_t)(ggml_get_op_params_i32(dst, 0)); - uint32_t s1 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 1) : s0; - uint32_t p0 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 2) : 0; - uint32_t p1 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 3) : 0; - uint32_t d0 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 4) : 1; - uint32_t d1 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 5) : 1; - - // tile-aligned shapes let the shader skip bounds checks - const uint32_t Cin = (uint32_t)src1->ne[2]; - const uint32_t CRS = Cin * KW * KH; - const uint32_t BS_K = vk_conv_block_sizes[shape].K; - const uint32_t BS_CRS = vk_conv_block_sizes[shape].CRS; - const uint32_t BS_NPQ = vk_conv_block_sizes[shape].NPQ; - const uint32_t aligned = ((K % BS_K == 0) && - (CRS % BS_CRS == 0) && - (NPQ % BS_NPQ == 0)) ? 1u : 0u; - - vk_conv2d_pipeline_state conv2d_pipeline_state(s0, s1, p0, p1, d0, d1, KW, KH, aligned); - - std::map<vk_conv2d_pipeline_state, vk_pipeline> *pipelines = nullptr; - if (op == GGML_OP_CONV_2D) { - if (src0->type == GGML_TYPE_F32) { - pipelines = &ctx->device->pipeline_conv2d_f32[shape]; - } else if (src0->type == GGML_TYPE_F16) { - pipelines = &ctx->device->pipeline_conv2d_f16_f32[shape]; - } - } else if (op == GGML_OP_CONV_TRANSPOSE_2D) { - if (src0->type == GGML_TYPE_F32) { - pipelines = &ctx->device->pipeline_conv_transpose_2d_f32[shape]; - } else if (src0->type == GGML_TYPE_F16) { - pipelines = &ctx->device->pipeline_conv_transpose_2d_f16_f32[shape]; - } - } - - vk_pipeline pipeline = nullptr; - - { - std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); - auto it = pipelines->find(conv2d_pipeline_state); - if (it != pipelines->end()) { - pipeline = it->second; - } else { - (*pipelines)[conv2d_pipeline_state] = pipeline = std::make_shared<vk_pipeline_struct>(); - } - } - - return pipeline; - } - return nullptr; - case GGML_OP_CONV_2D_DW: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - if (ggml_is_contiguous(src1)) { - return ctx->device->pipeline_conv2d_dw_whcn_f32; - } else if (ggml_is_contiguous_channels(src1)) { - return ctx->device->pipeline_conv2d_dw_cwhn_f32; - } - } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) { - if (ggml_is_contiguous(src1)) { - return ctx->device->pipeline_conv2d_dw_whcn_f16_f32; - } else if (ggml_is_contiguous_channels(src1)) { - return ctx->device->pipeline_conv2d_dw_cwhn_f16_f32; - } - } - return nullptr; - case GGML_OP_CONV_3D: - if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - const uint32_t OC = (uint32_t)ggml_get_op_params_i32(dst, 11); - const uint32_t IC = (uint32_t)ggml_get_op_params_i32(dst, 9); - const uint32_t N = (uint32_t)ggml_get_op_params_i32(dst, 10); - const uint32_t NPQ = N * dst->ne[2] * dst->ne[1] * dst->ne[0]; - const vk_conv_shapes shape = ggml_vk_conv_select_shape(ctx, OC, NPQ); - - const uint32_t KW = (uint32_t)src0->ne[0]; - const uint32_t KH = (uint32_t)src0->ne[1]; - const uint32_t KD = (uint32_t)src0->ne[2]; - const uint32_t s0 = (uint32_t)ggml_get_op_params_i32(dst, 0); - const uint32_t s1 = (uint32_t)ggml_get_op_params_i32(dst, 1); - const uint32_t s2 = (uint32_t)ggml_get_op_params_i32(dst, 2); - const uint32_t p0 = (uint32_t)ggml_get_op_params_i32(dst, 3); - const uint32_t p1 = (uint32_t)ggml_get_op_params_i32(dst, 4); - const uint32_t p2 = (uint32_t)ggml_get_op_params_i32(dst, 5); - const uint32_t d0 = (uint32_t)ggml_get_op_params_i32(dst, 6); - const uint32_t d1 = (uint32_t)ggml_get_op_params_i32(dst, 7); - const uint32_t d2 = (uint32_t)ggml_get_op_params_i32(dst, 8); - - const uint32_t CRS = IC * KW * KH * KD; - const uint32_t BS_K = vk_conv_block_sizes[shape].K; - const uint32_t BS_CRS = vk_conv_block_sizes[shape].CRS; - const uint32_t BS_NPQ = vk_conv_block_sizes[shape].NPQ; - const uint32_t aligned = ((OC % BS_K == 0) && - (CRS % BS_CRS == 0) && - (NPQ % BS_NPQ == 0)) ? 1u : 0u; - - vk_conv3d_pipeline_state conv3d_pipeline_state(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned); - - std::map<vk_conv3d_pipeline_state, vk_pipeline> *pipelines = nullptr; - if (src0->type == GGML_TYPE_F32) { - pipelines = &ctx->device->pipeline_conv3d_f32[shape]; - } else if (src0->type == GGML_TYPE_F16) { - pipelines = &ctx->device->pipeline_conv3d_f16_f32[shape]; - } else { - return nullptr; - } - - vk_pipeline pipeline = nullptr; - - { - std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); - auto it = pipelines->find(conv3d_pipeline_state); - if (it != pipelines->end()) { - pipeline = it->second; - } else { - (*pipelines)[conv3d_pipeline_state] = pipeline = std::make_shared<vk_pipeline_struct>(); - } - } - - return pipeline; - } - return nullptr; - case GGML_OP_ADD1: - if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_add1_f16_f16; - } - if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_add1_f16_f32; - } - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_add1_f32_f32; - } - return nullptr; - case GGML_OP_ARANGE: - if (dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_arange_f32; - } - return nullptr; - case GGML_OP_FILL: - if (dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_fill_f32; - } - if (dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_fill_f16; - } - return nullptr; - default: - return nullptr; - } - - GGML_UNUSED(src2); -} - -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_unary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); - const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - - p.misalign_offsets = (a_offset << 16) | d_offset; - - GGML_UNUSED(src1); - GGML_UNUSED(src2); - GGML_UNUSED(src3); -} - -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_sinkhorn_norm_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - p.a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); - p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - - GGML_UNUSED(src1); - GGML_UNUSED(src2); - GGML_UNUSED(src3); -} - -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_glu_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); - const uint32_t b_offset = src1 ? get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type) : a_offset; - const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - - GGML_ASSERT(a_offset < (1u << 8)); - GGML_ASSERT(b_offset < (1u << 8)); - GGML_ASSERT(d_offset < (1u << 8)); - - p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset; - - GGML_UNUSED(src2); - GGML_UNUSED(src3); -} - -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_sum_rows_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); - const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - - p.misalign_offsets = (a_offset << 16) | d_offset; - - GGML_UNUSED(src1); - GGML_UNUSED(src2); - GGML_UNUSED(src3); -} - -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_pad_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); - const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - - p.misalign_offsets = (a_offset << 16) | d_offset; - - GGML_UNUSED(src1); - GGML_UNUSED(src2); - GGML_UNUSED(src3); -} - -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_im2col_3d_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - const uint32_t a_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); - const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - - p.misalign_offsets = (a_offset << 16) | d_offset; - - GGML_UNUSED(src0); - GGML_UNUSED(src2); - GGML_UNUSED(src3); -} - -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_binary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); - const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type); - const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - - GGML_ASSERT(dst->op != GGML_OP_GET_ROWS || (a_offset == 0 && b_offset == 0 && d_offset == 0)); - - p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset; - - GGML_UNUSED(src2); - GGML_UNUSED(src3); -} - -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_concat_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type); - const uint32_t a_offset = get_misalign_bytes(ctx, src0) / unit_size; - const uint32_t b_offset = get_misalign_bytes(ctx, src1) / unit_size; - const uint32_t d_offset = get_misalign_bytes(ctx, dst) / unit_size; - - p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset; - - GGML_UNUSED(src2); - GGML_UNUSED(src3); -} - -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_upscale_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); - const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - - p.a_offset = a_offset; - p.d_offset = d_offset; - - GGML_UNUSED(src1); - GGML_UNUSED(src2); - GGML_UNUSED(src3); -} - -template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_rope_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { - p.a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); - p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - - GGML_UNUSED(src1); - GGML_UNUSED(src2); - GGML_UNUSED(src3); -} - -template<typename PC> -static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst, ggml_op op, PC&& pc) { - VK_LOG_DEBUG("ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; - if (src1 != nullptr) { - std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; - } - if (src2 != nullptr) { - std::cerr << "), (" << src2 << ", name=" << src2->name << ", type=" << src2->type << ", ne0=" << src2->ne[0] << ", ne1=" << src2->ne[1] << ", ne2=" << src2->ne[2] << ", ne3=" << src2->ne[3] << ", nb0=" << src2->nb[0] << ", nb1=" << src2->nb[1] << ", nb2=" << src2->nb[2] << ", nb3=" << src2->nb[3]; - } - if (src3 != nullptr) { - std::cerr << "), (" << src3 << ", name=" << src3->name << ", type=" << src3->type << ", ne0=" << src3->ne[0] << ", ne1=" << src3->ne[1] << ", ne2=" << src3->ne[2] << ", ne3=" << src3->ne[3] << ", nb0=" << src3->nb[0] << ", nb1=" << src3->nb[1] << ", nb2=" << src3->nb[2] << ", nb3=" << src3->nb[3]; - } - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; - std::cerr << "), " << ggml_op_name(op) << ")"); - GGML_ASSERT(op == GGML_OP_GET_ROWS || op == GGML_OP_CPY || op == GGML_OP_CONCAT || (!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type)))); // NOLINT - GGML_ASSERT(dst->buffer != nullptr); - const uint64_t ne00 = src0->ne[0]; - const uint64_t ne01 = src0->ne[1]; - const uint64_t ne02 = src0->ne[2]; - const uint64_t ne03 = src0->ne[3]; - - const bool use_src1 = src1 != nullptr; - const uint64_t ne10 = use_src1 ? src1->ne[0] : 0; - const uint64_t ne11 = use_src1 ? src1->ne[1] : 0; - const uint64_t ne12 = use_src1 ? src1->ne[2] : 0; - const uint64_t ne13 = use_src1 ? src1->ne[3] : 0; - - const bool use_src2 = src2 != nullptr; - const bool use_src3 = src3 != nullptr; - - init_pushconst_fastdiv(pc); - - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op); - - if (pipeline == nullptr) { - std::cerr << "ggml_vulkan: Error: Missing op: " << ggml_op_name(op) << " for " << ggml_type_name(src0->type); - if (src1 != nullptr) { - std::cerr << " and " << ggml_type_name(src1->type); - } - std::cerr << " to " << ggml_type_name(dst->type) << std::endl; - GGML_ABORT("fatal error"); - } - - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - - vk_subbuffer src0_buf = ggml_vk_tensor_subbuffer(ctx, src0, true); - vk_subbuffer src1_buf = use_src1 ? ggml_vk_tensor_subbuffer(ctx, src1, true) : vk_subbuffer{}; - vk_subbuffer src2_buf = use_src2 ? ggml_vk_tensor_subbuffer(ctx, src2, true) : vk_subbuffer{}; - vk_subbuffer src3_buf = use_src3 ? ggml_vk_tensor_subbuffer(ctx, src3, true) : vk_subbuffer{}; - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); - - // Compute misalignment offset for descriptors and store it in in push constants. - init_pushconst_tensor_offsets(ctx, pc, src0, src1, src2, src3, dst); - - std::array<uint32_t, 3> elements; - - switch (op) { - case GGML_OP_NORM: - case GGML_OP_RMS_NORM_BACK: - case GGML_OP_L2_NORM: - case GGML_OP_SOFT_MAX: - case GGML_OP_SOFT_MAX_BACK: - case GGML_OP_SUM_ROWS: - case GGML_OP_CUMSUM: - case GGML_OP_MEAN: - case GGML_OP_ARGMAX: - { - const uint32_t nr = ggml_nrows(src0); - if (nr > 262144) { - elements = { 512, 512, CEIL_DIV(nr, 262144) }; - } else if (nr > 512) { - elements = { 512, CEIL_DIV(nr, 512), 1 }; - } else { - elements = { nr, 1, 1 }; - } - } break; - case GGML_OP_SINKHORN_NORM: - { - // One invocation per TOKEN, not per row: the shader holds the whole - // n*n matrix for a token in registers. ggml_nrows() here would be - // n*nt and dispatch n x more invocations than exist tokens, all of - // which return immediately on the t >= nt guard. - const uint32_t nt = (uint32_t)(ne02 * ne03); - if (nt > 262144) { - elements = { 512, 512, CEIL_DIV(nt, 262144) }; - } else if (nt > 512) { - elements = { 512, CEIL_DIV(nt, 512), 1 }; - } else { - elements = { nt, 1, 1 }; - } - } break; - case GGML_OP_SOLVE_TRI: - { - uint32_t nr = (uint32_t)(ne02 * ne03); - if (nr > 262144) { - elements = { 512, 512, CEIL_DIV(nr, 262144) }; - } else if (nr > 512) { - elements = { 512, CEIL_DIV(nr, 512), 1 }; - } else { - elements = { nr, 1, 1 }; - } - } - break; - case GGML_OP_RMS_NORM: - if (ctx->do_add_rms_partials) { - // Run one element per thread, 128 threads per workgroup - elements = { (uint32_t)CEIL_DIV(ne00, 128), 1, 1 }; - } else { - elements = { (uint32_t)ne01, (uint32_t)ne02, (uint32_t)ne03 }; - } - break; - - case GGML_OP_SUM: - // We use GGML_OP_SUM_ROWS with 1 row. - elements = { 1, 1, 1 }; - break; - case GGML_OP_GROUP_NORM: - { - const uint32_t num_groups = dst->op_params[0]; - elements = { num_groups * (uint32_t)src0->ne[3], 1, 1 }; - } break; - case GGML_OP_DIAG_MASK_INF: - elements = { (uint32_t)ggml_nrows(src0), (uint32_t)ne00, 1 }; - break; - case GGML_OP_ROPE: - case GGML_OP_ROPE_BACK: - { - uint32_t nrows = (uint32_t)ggml_nrows(src0); - uint32_t z = 1; - if (nrows > ctx->device->properties.limits.maxComputeWorkGroupCount[0]) { - z = CEIL_DIV(nrows, 32768); - nrows = 32768; - } - elements = { nrows, (uint32_t)ne00, z }; - - } break; - case GGML_OP_GET_ROWS: - elements = { (uint32_t)ne00, (uint32_t)ne10, (uint32_t)(ne11 * ne12) }; - elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); - elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); - break; - case GGML_OP_GET_ROWS_BACK: - elements = { (uint32_t)dst->ne[0], (uint32_t)dst->ne[1], 1 }; - elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); - break; - case GGML_OP_ARGSORT: - GGML_ASSERT(0); - break; - case GGML_OP_IM2COL: - { - const bool is_2D = dst->op_params[6] == 1; - - const uint32_t IC = src1->ne[is_2D ? 2 : 1]; - - const uint32_t KH = is_2D ? src0->ne[1] : 1; - const uint32_t KW = src0->ne[0]; - - const uint32_t OH = is_2D ? dst->ne[2] : 1; - const uint32_t OW = dst->ne[1]; - - const uint32_t batch = src1->ne[is_2D ? 3 : 2]; - - const uint32_t CHW = IC * KH * KW; - // Cap X workgroups to limit concurrent IC channel reads. - // The shader loops over X to cover the full CHW dimension. - // AMD prefers a lower limit - const uint32_t min_cap = ctx->device->vendor_id == VK_VENDOR_ID_AMD ? 512u : 4096u; - const uint32_t x_elements = std::min(CHW, std::max(min_cap, OW * KH * KW)); - elements = { x_elements, OW, OH * batch }; - elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); - elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); - } break; - case GGML_OP_IM2COL_3D: - { - const uint32_t IC = ((const uint32_t *)(dst->op_params))[9]; - - const uint32_t N = ne13 / IC; - - const uint32_t KD = ne02; - const uint32_t KH = ne01; - const uint32_t KW = ne00; - - const uint32_t OD = dst->ne[3] / N; - const uint32_t OH = dst->ne[2]; - const uint32_t OW = dst->ne[1]; - - const uint32_t IC_KD_KH_KW = IC*KD*KH*KW; - const uint32_t N_OD_OH = N*OD*OH; - - elements = { IC_KD_KH_KW, OW, N_OD_OH }; - elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); - } break; - case GGML_OP_TIMESTEP_EMBEDDING: - { - const uint32_t dim = dst->op_params[0]; - uint32_t half_ceil = (dim + 1) / 2; - elements = { half_ceil, (uint32_t)src0->ne[0], 1 }; - } break; - case GGML_OP_CONV_TRANSPOSE_1D: - { - elements = {uint32_t(src0->ne[1]), 1, 1}; // parallelize in {Cout, 1, 1} - } break; - case GGML_OP_COL2IM_1D: - { - elements = { uint32_t(dst->ne[0]), uint32_t(dst->ne[1]), 1 }; - } break; - case GGML_OP_POOL_1D: - { - const uint32_t N = dst->ne[3] * dst->ne[2]; - const uint32_t OC = dst->ne[1]; - const uint32_t OL = dst->ne[0]; - elements = { N * OC * OL, 1, 1}; - } break; - case GGML_OP_POOL_2D: - { - const uint32_t N = dst->ne[3]; - const uint32_t OC = dst->ne[2]; - const uint32_t OH = dst->ne[1]; - const uint32_t OW = dst->ne[0]; - elements = { N * OC * OH * OW, 1, 1}; - } break; - case GGML_OP_CONV_2D: - case GGML_OP_CONV_TRANSPOSE_2D: - if constexpr (std::is_same_v<PC, vk_op_conv2d_push_constants>) { - const uint32_t NPQ = pc.N * pc.OH * pc.OW; - const vk_conv_shapes shape = ggml_vk_conv_select_shape(ctx, pc.Cout, NPQ); - const uint32_t NPQ_blocks = CEIL_DIV(NPQ, vk_conv_block_sizes[shape].NPQ); - - elements = { pc.Cout, NPQ_blocks, 1 }; - if (elements[1] > 512) { - elements[2] = CEIL_DIV(elements[1], 512); - elements[1] = 512; - } - } else { - GGML_ABORT("invalid push constant type for CONV_2D"); - } - break; - case GGML_OP_CONV_3D: - if constexpr (std::is_same_v<PC, vk_op_conv3d_push_constants>) { - const uint32_t NPQ = pc.N * pc.OD * pc.OH * pc.OW; - const vk_conv_shapes shape = ggml_vk_conv_select_shape(ctx, pc.OC, NPQ); - const uint32_t NPQ_blocks = CEIL_DIV(NPQ, vk_conv_block_sizes[shape].NPQ); - - elements = { pc.OC, NPQ_blocks, 1 }; - if (elements[1] > 512) { - elements[2] = CEIL_DIV(elements[1], 512); - elements[1] = 512; - } - } else { - GGML_ABORT("invalid push constant type for CONV_3D"); - } - break; - case GGML_OP_ADD: - case GGML_OP_SUB: - case GGML_OP_DIV: - case GGML_OP_MUL: - case GGML_OP_ADD1: - case GGML_OP_OUT_PROD: - case GGML_OP_ARANGE: - case GGML_OP_FILL: - case GGML_OP_SCALE: - case GGML_OP_SQR: - case GGML_OP_SQRT: - case GGML_OP_SIN: - case GGML_OP_COS: - case GGML_OP_LOG: - case GGML_OP_TRI: - case GGML_OP_DIAG: - case GGML_OP_CLAMP: - case GGML_OP_LEAKY_RELU: - case GGML_OP_PAD: - case GGML_OP_PAD_REFLECT_1D: - case GGML_OP_ROLL: - case GGML_OP_REPEAT: - case GGML_OP_REPEAT_BACK: - case GGML_OP_CPY: - case GGML_OP_CONCAT: - case GGML_OP_UPSCALE: - case GGML_OP_UNARY: - case GGML_OP_GLU: - case GGML_OP_CONV_2D_DW: - { - uint32_t ne = ggml_nelements(dst); - if (op == GGML_OP_CPY && ggml_is_quantized(src0->type) && ggml_is_quantized(dst->type)) { - // Convert from number of logical elements to 2- or 4-byte units. - ne /= ggml_blck_size(src0->type); - if ((ggml_type_size(src0->type) % 4) == 0) { - ne *= ggml_type_size(src0->type) / 4; - } else { - ne *= ggml_type_size(src0->type) / 2; - } - } - if (op == GGML_OP_CONCAT && ggml_is_quantized(dst->type)) { - ne = ne / ggml_blck_size(dst->type) * ggml_type_size(dst->type) / ggml_vk_concat_unit_size(dst->type); - } - // copy_to_quant has block size of 32, and each thread does QUANT_K elements. - // Splitting into 512x512xZ wouldn't work well since each workgroup does 1024 elements. - // So divide by block size here before splitting into 512x512 groups. - if (op == GGML_OP_CPY && !ggml_is_quantized(src0->type) && ggml_is_quantized(dst->type)) { - ne = CEIL_DIV(ne, ggml_blck_size(dst->type)); - } - if (ne > 262144) { - elements = { 512, 512, CEIL_DIV(ne, 262144) }; - } else if (ne > 512) { - elements = { 512, CEIL_DIV(ne, 512), 1 }; - } else { - elements = { ne, 1, 1 }; - } - - if (pipeline == ctx->device->pipeline_cpy_transpose_02_32 || - pipeline == ctx->device->pipeline_cpy_transpose_02_16) { - // 32x32 tiles over dims 0 and 2; dim1 and dim3 are the batch - elements[0] = (uint32_t)CEIL_DIV(dst->ne[0], 32); - elements[1] = (uint32_t)CEIL_DIV(dst->ne[2], 32); - elements[2] = (uint32_t)(dst->ne[1]*dst->ne[3]); - elements[0] = std::min(elements[0], ctx->device->properties.limits.maxComputeWorkGroupCount[0]); - elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); - elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); - } else if (pipeline == ctx->device->pipeline_cpy_transpose_32 || - pipeline == ctx->device->pipeline_cpy_transpose_16) { - // 32x32 tiles - elements[0] = (uint32_t)CEIL_DIV(dst->ne[0], 32); - elements[1] = (uint32_t)CEIL_DIV(dst->ne[1], 32); - elements[2] = (uint32_t)(dst->ne[2]*dst->ne[3]); - elements[0] = std::min(elements[0], ctx->device->properties.limits.maxComputeWorkGroupCount[0]); - elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); - elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); - } - } break; - case GGML_OP_ADD_ID: - { - elements = { (uint32_t)ne01, (uint32_t)ne02, 1 }; - } break; - case GGML_OP_SET_ROWS: - { - uint32_t ne = ggml_nelements(src0); - if (dst->type == GGML_TYPE_TURBO4_0) { - // turbo4_0 uses a bespoke COOPERATIVE shader: one workgroup (128 threads) - // per QUANT_K=128 block, so dispatch one workgroup per block. - ne = CEIL_DIV(ne, ggml_blck_size(dst->type)); - } else if (ggml_is_quantized(dst->type)) { - // quants run 32 threads each doing QUANT_K elements - ne = CEIL_DIV(ne, 32 * ggml_blck_size(dst->type)); - } else { - // scalar types do one element per thread, running 512 threads - ne = CEIL_DIV(ne, 512); - } - if (ne > 262144) { - elements = { 512, 512, CEIL_DIV(ne, 262144) }; - } else if (ne > 512) { - elements = { 512, CEIL_DIV(ne, 512), 1 }; - } else { - elements = { ne, 1, 1 }; + case GGML_OP_UPSCALE: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + uint32_t mode = (ggml_get_op_params_i32(dst, 0) & (0xFF | GGML_SCALE_FLAG_ANTIALIAS)); + switch (mode) { + case GGML_SCALE_MODE_NEAREST: + return ctx->device->pipeline_upscale_nearest_f32; + case GGML_SCALE_MODE_BILINEAR: + return ctx->device->pipeline_upscale_bilinear_f32; + case GGML_SCALE_MODE_BICUBIC: + return ctx->device->pipeline_upscale_bicubic_f32; + case GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS: + return ctx->device->pipeline_upscale_bilinear_antialias_f32; + default: + return nullptr; } } - break; - case GGML_OP_SSM_CONV: - { - const uint32_t nr = src0->ne[1]; - const uint32_t n_t = dst->ne[1]; - const uint32_t n_s = dst->ne[2]; - elements = { nr, n_t, n_s }; + return nullptr; + case GGML_OP_SCALE: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_scale_f32; } - break; - default: - elements = { (uint32_t)ggml_nelements(src0), 1, 1 }; - break; - } - - if (op == GGML_OP_ADD || op == GGML_OP_RMS_NORM) { - vk_subbuffer a_buf = src0_buf; - if (ctx->do_add_rms_partials) { - a_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_add_rms_partials, ctx->prealloc_size_add_rms_partials_offset); + return nullptr; + case GGML_OP_SQR: + if (src0->type == dst->type && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { + return ctx->device->pipeline_sqr[dst->type == GGML_TYPE_F16]; } - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - { src0_buf, src1_buf, dst_buf, a_buf }, pc, elements); - } else if (op == GGML_OP_GLU) { - // Empty src1 is possible in glu, but the shader needs a buffer - vk_subbuffer subbuf1 = use_src1 ? src1_buf : src0_buf; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, subbuf1, dst_buf }, pc, elements); - } else if (op == GGML_OP_SOFT_MAX) { - // Empty src1 and src2 is possible in soft_max, but the shader needs a buffer - vk_subbuffer subbuf1 = use_src1 ? src1_buf : src0_buf; - vk_subbuffer subbuf2 = use_src2 ? src2_buf : src0_buf; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, subbuf1, subbuf2, dst_buf }, pc, elements); - } else if (op == GGML_OP_ROPE || op == GGML_OP_ROPE_BACK) { - // Empty src2 and src3 is possible in rope, but the shader needs a buffer - vk_subbuffer subbuf2 = use_src2 ? src2_buf : src0_buf; - vk_subbuffer subbuf3 = use_src3 ? src3_buf : src0_buf; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, subbuf2, dst_buf, subbuf3 }, pc, elements); - } else if (op == GGML_OP_IM2COL || op == GGML_OP_IM2COL_3D) { - if (ctx->device->shader_int64 && ctx->device->buffer_device_address) { - // buffer device address path doesn't use dst buffer - dst_buf.size = 1; + return nullptr; + case GGML_OP_SQRT: + if (src0->type == dst->type && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { + return ctx->device->pipeline_sqrt[dst->type == GGML_TYPE_F16]; } - // im2col uses only src1 and dst buffers - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src1_buf, dst_buf }, pc, elements); - } else if (op == GGML_OP_COUNT_EQUAL) { - // count_equal assumes that destination buffer is initialized with zeroes - ggml_vk_buffer_memset_async(subctx, dst_buf.buffer, dst_buf.offset, 0, dst_buf.size); - ggml_vk_sync_buffers(ctx, subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, dst_buf }, pc, elements); - } else if (op == GGML_OP_OPT_STEP_SGD) { - // OPT_STEP_SGD works on src0, it does not need dst - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, src2_buf }, pc, elements); - } else if (use_src3) { - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, src2_buf, src3_buf, dst_buf }, pc, elements); - } else if (use_src2) { - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, src2_buf, dst_buf }, pc, elements); - } else if (use_src1) { - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, dst_buf }, pc, elements); - } else { - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, dst_buf }, pc, elements); - } -} - -static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_GET_ROWS, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - 0.0f, 0.0f, 0, - }); -} - -static void ggml_vk_get_rows_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_GET_ROWS_BACK, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2], (uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - 0.0f, 0.0f, 0, - }); -} - -static void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - int nb1 = dst->op_params[0] / src0_type_size; // 4 bytes of float32 - int nb2 = dst->op_params[1] / src0_type_size; // 4 bytes of float32 - int nb3 = dst->op_params[2] / src0_type_size; // 4 bytes of float32 - int offset = dst->op_params[3] / src0_type_size; // offset in bytes - - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, dst->op, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, - 0, - 0.0f, 0.0f, offset, - }); -} - -static void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) { - const ggml_tensor *first_node = cgraph->nodes[node_idx]; - const ggml_tensor *dst = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; - - // Make a list of all the tensors used by the op. - // Last element of the list is the dest tensor. - const ggml_tensor *tensors[MAX_PARAMETER_COUNT]; - uint32_t num_srcs = ctx->num_additional_fused_ops + 2; - uint32_t num_tensors = num_srcs + 1; - GGML_ASSERT(num_tensors + ctx->do_add_rms_partials <= MAX_PARAMETER_COUNT); - - tensors[0] = first_node->src[0]; - tensors[1] = first_node->src[1]; - for (int32_t i = 0; i < ctx->num_additional_fused_ops; ++i) { - // check whether the previous result is src[0] or src[1] - if (cgraph->nodes[node_idx + i] == cgraph->nodes[node_idx + i + 1]->src[0]) { - tensors[i+2] = cgraph->nodes[node_idx + i + 1]->src[1]; - } else { - tensors[i+2] = cgraph->nodes[node_idx + i + 1]->src[0]; + return nullptr; + case GGML_OP_SIN: + if (src0->type == dst->type && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { + return ctx->device->pipeline_sin[dst->type == GGML_TYPE_F16]; } - } - tensors[num_srcs] = dst; - - vk_op_multi_add_push_constants pc; - pc.ne20 = (uint32_t)dst->ne[0]; - pc.ne21 = (uint32_t)dst->ne[1]; - pc.ne22 = (uint32_t)dst->ne[2]; - pc.ne23 = (uint32_t)dst->ne[3]; - - for (uint32_t i = 0; i < num_tensors; ++i) { - const ggml_tensor *t = tensors[i]; - pc.nb[i][0] = (uint32_t)t->nb[0] / sizeof(float); - pc.nb[i][1] = (uint32_t)t->nb[1] / sizeof(float); - pc.nb[i][2] = (uint32_t)t->nb[2] / sizeof(float); - pc.nb[i][3] = (uint32_t)t->nb[3] / sizeof(float); - } - pc.rms_partials = ctx->do_add_rms_partials; - - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, tensors[0], tensors[1], nullptr, dst, dst->op); - - if (pipeline == nullptr) { - std::cerr << "ggml_vulkan: Error: Missing multi_add"; - GGML_ABORT("fatal error"); - } - - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - - ggml_backend_vk_buffer_context * buf_ctx[MAX_PARAMETER_COUNT]; - vk_buffer buf[MAX_PARAMETER_COUNT]; - size_t offset[MAX_PARAMETER_COUNT]; - bool uma[MAX_PARAMETER_COUNT]; - - for (uint32_t i = 0; i < num_tensors; ++i) { - buf_ctx[i] = (ggml_backend_vk_buffer_context *)tensors[i]->buffer->context; - buf[i] = nullptr; - offset[i] = 0; - uma[i] = false; - - if (ctx->device->uma) { - ggml_vk_host_get(ctx->device, tensors[i]->data, buf[i], offset[i]); - uma[i] = buf[i] != nullptr; + return nullptr; + case GGML_OP_COS: + if (src0->type == dst->type && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { + return ctx->device->pipeline_cos[dst->type == GGML_TYPE_F16]; + } + return nullptr; + case GGML_OP_LOG: + if (src0->type == dst->type && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { + return ctx->device->pipeline_log[dst->type == GGML_TYPE_F16]; + } + return nullptr; + case GGML_OP_TRI: + if (src0->type == dst->type && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { + return ctx->device->pipeline_tri[dst->type == GGML_TYPE_F16]; } - if (!uma[i]) { - buf[i] = buf_ctx[i]->dev_buffer; - offset[i] = vk_tensor_offset(tensors[i]) + tensors[i]->view_offs; + return nullptr; + case GGML_OP_DIAG: + if (src0->type == dst->type && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { + return ctx->device->pipeline_diag[dst->type == GGML_TYPE_F16]; } - GGML_ASSERT(buf[i] != nullptr); - } - // If any remaining descriptors are unused, just point them at src[0] - for (uint32_t i = num_tensors; i < MAX_PARAMETER_COUNT; ++i) { - buf[i] = buf[0]; - offset[i] = 0; - } - if (ctx->do_add_rms_partials) { - buf[num_tensors] = ctx->prealloc_add_rms_partials; - offset[num_tensors] = ctx->prealloc_size_add_rms_partials_offset; - } - - std::array<uint32_t, 3> elements; - - uint32_t ne = ggml_nelements(dst); - if (ne > 262144) { - elements = { 512, 512, CEIL_DIV(ne, 262144) }; - } else if (ne > 512) { - elements = { 512, CEIL_DIV(ne, 512), 1 }; - } else { - elements = { ne, 1, 1 }; - } - - static_assert(MAX_PARAMETER_COUNT == 12); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + return nullptr; + case GGML_OP_CLAMP: + if (src0->type == dst->type && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { + return ctx->device->pipeline_clamp[dst->type == GGML_TYPE_F16]; + } + return nullptr; + case GGML_OP_PAD: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_pad_f32; + } + return nullptr; + case GGML_OP_PAD_REFLECT_1D: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_pad_reflect_1d_f32; + } + return nullptr; + case GGML_OP_ROLL: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_roll_f32; + } + return nullptr; + case GGML_OP_REPEAT: + if (ggml_type_size(src0->type) == sizeof(float) && ggml_type_size(dst->type) == sizeof(float)) { + return ctx->device->pipeline_repeat_i32; + } + if (ggml_type_size(src0->type) == 2 && ggml_type_size(dst->type) == 2) { + return ctx->device->pipeline_repeat_i16; + } + return nullptr; + case GGML_OP_REPEAT_BACK: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_repeat_back_f32; + } + return nullptr; + case GGML_OP_CPY: + case GGML_OP_CONT: + case GGML_OP_DUP: + return ggml_vk_get_cpy_pipeline(ctx, src0, dst, dst->type); + case GGML_OP_SET_ROWS: { - ggml_vk_subbuffer(ctx, buf[0], offset[0]), - ggml_vk_subbuffer(ctx, buf[1], offset[1]), - ggml_vk_subbuffer(ctx, buf[2], offset[2]), - ggml_vk_subbuffer(ctx, buf[3], offset[3]), - ggml_vk_subbuffer(ctx, buf[4], offset[4]), - ggml_vk_subbuffer(ctx, buf[5], offset[5]), - ggml_vk_subbuffer(ctx, buf[6], offset[6]), - ggml_vk_subbuffer(ctx, buf[7], offset[7]), - ggml_vk_subbuffer(ctx, buf[8], offset[8]), - ggml_vk_subbuffer(ctx, buf[9], offset[9]), - ggml_vk_subbuffer(ctx, buf[10], offset[10]), - ggml_vk_subbuffer(ctx, buf[11], offset[11]), - }, pc, elements); -} - -static void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ADD, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - 0.0f, 0.0f, ctx->do_add_rms_partials, - }); -} - -static void ggml_vk_out_prod(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_OUT_PROD, { - (uint32_t)ggml_nelements(dst), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], - (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], - (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], - (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - 0.0f, 0.0f, 0, - }); -} - -static void ggml_vk_sub(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SUB, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - 0.0f, 0.0f, 0, - }); -} - -static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_MUL, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - 0.0f, 0.0f, 0, - }); -} - -static void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_DIV, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - 0.0f, 0.0f, 0, - }); -} - -static void ggml_vk_add_id(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t src2_type_size = ggml_type_size(src2->type); - - ggml_vk_op_f32<vk_op_add_id_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_ADD_ID, { - (uint32_t)dst->ne[0], - (uint32_t)dst->ne[1], - (uint32_t)src0->nb[1] / src0_type_size, - (uint32_t)src0->nb[2] / src0_type_size, - (uint32_t)src1->nb[1] / src1_type_size, - (uint32_t)src2->nb[1] / src2_type_size, - }); -} - -static void ggml_vk_op_f32_wkv(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, const vk_op_rwkv_wkv6_push_constants&& pc, int version) { - GGML_ASSERT(version == 6 || version == 7); - int num_srcs = version == 6 ? 6 : 7; - - for (int i = 0; i < num_srcs; i++) { - GGML_ASSERT(!ggml_is_quantized(dst->src[i]->type)); - } - - GGML_ASSERT(dst->buffer != nullptr); - - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op); - GGML_ASSERT(pipeline != nullptr); - - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); - vk_subbuffer src_buf[7] = {}; - for (int i = 0; i < num_srcs; i++) { - src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]); - } - - std::array<uint32_t, 3> elements = { - (uint32_t)(pc.B * pc.H), - 1, - 1 - }; - - if (version == 6) { - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf}, - pc, elements); - } else if (version == 7) { - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], src_buf[6], dst_buf}, - pc, elements); - } else { - // shouldn't happen - GGML_ASSERT(false); - } -} + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) { + return nullptr; + } + const int src_idx = src0->type == GGML_TYPE_F16; + if (src1->type == GGML_TYPE_I64) { + return ctx->device->pipeline_set_rows_i64[src_idx][dst->type]; + } else if (src1->type == GGML_TYPE_I32) { + return ctx->device->pipeline_set_rows_i32[src_idx][dst->type]; + } + return nullptr; + } + case GGML_OP_SILU_BACK: + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_silu_back_f32; + } + return nullptr; + case GGML_OP_NORM: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_norm_f32; + } + return nullptr; + case GGML_OP_GROUP_NORM: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_group_norm_f32; + } + return nullptr; + case GGML_OP_RMS_NORM: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + if (ctx->do_add_rms_partials) { + return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_partials_f32 : ctx->device->pipeline_rms_norm_partials_f32; + } + return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32; + } + return nullptr; + case GGML_OP_RMS_NORM_BACK: + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_rms_norm_back_f32; + } + return nullptr; + case GGML_OP_L2_NORM: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_l2_norm_f32; + } + return nullptr; + case GGML_OP_SINKHORN_NORM: + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return nullptr; + } + switch (src0->ne[0]) { + case 2: return ctx->device->pipeline_sinkhorn_norm_f32[0]; + case 4: return ctx->device->pipeline_sinkhorn_norm_f32[1]; + case 8: return ctx->device->pipeline_sinkhorn_norm_f32[2]; + default: return nullptr; + } + case GGML_OP_UNARY: + if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || + (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) || + (src0->type != dst->type)) { + return nullptr; + } -static void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - const size_t seq_length = dst->src[0]->ne[2]; - const size_t n_embed = dst->ne[0]; - const size_t n_heads = dst->src[0]->ne[1]; - const size_t n_seqs = dst->src[5]->ne[1]; + switch (ggml_get_unary_op(dst)) { + case GGML_UNARY_OP_EXP: + return ctx->device->pipeline_exp[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_EXPM1: + return ctx->device->pipeline_expm1[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_ELU: + return ctx->device->pipeline_elu[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_SILU: + return ctx->device->pipeline_silu[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_GELU: + return ctx->device->pipeline_gelu[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_GELU_ERF: + return ctx->device->pipeline_gelu_erf[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_GELU_QUICK: + return ctx->device->pipeline_gelu_quick[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_RELU: + return ctx->device->pipeline_relu[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_XIELU: + return ctx->device->pipeline_xielu[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_NEG: + return ctx->device->pipeline_neg[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_TANH: + return ctx->device->pipeline_tanh[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_SIGMOID: + return ctx->device->pipeline_sigmoid[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_HARDSIGMOID: + return ctx->device->pipeline_hardsigmoid[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_HARDSWISH: + return ctx->device->pipeline_hardswish[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_ABS: + return ctx->device->pipeline_abs[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_SOFTPLUS: + return ctx->device->pipeline_softplus[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_STEP: + return ctx->device->pipeline_step[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_ROUND: + return ctx->device->pipeline_round[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_CEIL: + return ctx->device->pipeline_ceil[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_FLOOR: + return ctx->device->pipeline_floor[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_TRUNC: + return ctx->device->pipeline_trunc[dst->type == GGML_TYPE_F16]; + case GGML_UNARY_OP_SGN: + return ctx->device->pipeline_sgn[dst->type == GGML_TYPE_F16]; + default: + break; + } + return nullptr; + case GGML_OP_GLU: + if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || + (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) || + (src0->type != dst->type)) { + return nullptr; + } - ggml_vk_op_f32_wkv( - ctx, subctx, dst, - { - (uint32_t)n_seqs, - (uint32_t)seq_length, - (uint32_t)n_embed, - (uint32_t)n_heads, - }, - 6 - ); -} + switch (ggml_get_glu_op(dst)) { + case GGML_GLU_OP_GEGLU: + return ctx->device->pipeline_geglu[dst->type == GGML_TYPE_F16]; + case GGML_GLU_OP_REGLU: + return ctx->device->pipeline_reglu[dst->type == GGML_TYPE_F16]; + case GGML_GLU_OP_SWIGLU: + return ctx->device->pipeline_swiglu[dst->type == GGML_TYPE_F16]; + case GGML_GLU_OP_SWIGLU_OAI: + return ctx->device->pipeline_swiglu_oai[dst->type == GGML_TYPE_F16]; + case GGML_GLU_OP_SWIGLU_CLAMP: + return ctx->device->pipeline_swiglu_clamp[dst->type == GGML_TYPE_F16]; + case GGML_GLU_OP_GEGLU_ERF: + return ctx->device->pipeline_geglu_erf[dst->type == GGML_TYPE_F16]; + case GGML_GLU_OP_GEGLU_QUICK: + return ctx->device->pipeline_geglu_quick[dst->type == GGML_TYPE_F16]; + default: + break; + } + return nullptr; + case GGML_OP_DIAG_MASK_INF: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_diag_mask_inf_f32; + } + return nullptr; + case GGML_OP_SOFT_MAX: + GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); + GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F32); -static void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - const size_t seq_length = dst->src[0]->ne[2]; - const size_t n_embed = dst->ne[0]; - const size_t n_heads = dst->src[0]->ne[1]; - const size_t n_seqs = dst->src[6]->ne[1]; + if (ctx->num_additional_fused_ops) { + uint32_t idx = (uint32_t)ceilf(log2f(float(dst->ne[0]))); + GGML_ASSERT(idx < num_topk_moe_pipelines); + // use n_experts from push constant if it's not equal to the power of two spec constant + bool use_push = dst->ne[0] != (1u << idx); + return ctx->device->pipeline_topk_moe[idx][use_push]; + } - ggml_vk_op_f32_wkv( - ctx, subctx, dst, + if (src0->type == GGML_TYPE_F32 && (src1 == nullptr || src1->type == GGML_TYPE_F32) && dst->type == GGML_TYPE_F32) { + return src0->ne[0] > 1024 ? ctx->device->pipeline_soft_max_f32_wg512 : ctx->device->pipeline_soft_max_f32; + } + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) { + return src0->ne[0] > 1024 ? ctx->device->pipeline_soft_max_f32_f16_wg512 : ctx->device->pipeline_soft_max_f32_f16; + } + return nullptr; + case GGML_OP_SOFT_MAX_BACK: + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_soft_max_back_f32; + } + return nullptr; + case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: { - (uint32_t)n_seqs, - (uint32_t)seq_length, - (uint32_t)n_embed, - (uint32_t)n_heads, - }, - 7 - ); -} + const ggml_tensor *rope = ctx->num_additional_fused_ops == 2 ? dst->src[0]->src[0] : dst; + const int mode = ((const int32_t *) rope->op_params)[2]; + const bool is_neox = mode & GGML_ROPE_TYPE_NEOX; + const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; + const bool is_vision = mode == GGML_ROPE_TYPE_VISION; -static void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - const size_t seq_length = dst->src[0]->ne[2]; - const size_t n_embed = dst->ne[0]; - const size_t n_heads = dst->src[0]->ne[1]; - const size_t n_seqs = dst->src[4]->ne[1]; + if (is_neox) { + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_rope_neox_f32; + } + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_rope_neox_f32_f16; + } + if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_rope_neox_f16; + } + } else if (is_mrope && !is_vision) { + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_rope_multi_f32; + } + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_rope_multi_f32_f16; + } + if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_rope_multi_f16; + } + } else if (is_vision) { + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_rope_vision_f32; + } + if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_rope_vision_f16; + } + } else { + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_rope_norm_f32; + } + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_rope_norm_f32_f16; + } + if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_rope_norm_f16; + } + } + return nullptr; + } + case GGML_OP_SUM: + case GGML_OP_SUM_ROWS: + case GGML_OP_MEAN: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_sum_rows_f32; + } + return nullptr; + case GGML_OP_CROSS_ENTROPY_LOSS: + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return src0->ne[0] > 1024 ? ctx->device->pipeline_cross_entropy_loss_f32_wg512 : ctx->device->pipeline_cross_entropy_loss_f32; + } + return nullptr; + case GGML_OP_CROSS_ENTROPY_LOSS_BACK: + // src0 is the scalar grad; src1 is logits + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && src2 && src2->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return src1->ne[0] > 1024 ? ctx->device->pipeline_cross_entropy_loss_back_f32_wg512 : ctx->device->pipeline_cross_entropy_loss_back_f32; + } + return nullptr; + case GGML_OP_CUMSUM: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + if (src0->ne[0] <= 512) { + return ctx->device->pipeline_cumsum_small_f32; + } else { + return ctx->device->pipeline_cumsum_f32; + } + } + return nullptr; + case GGML_OP_SOLVE_TRI: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - float scale; - memcpy(&scale, dst->op_params, sizeof(float)); + vk_solve_tri_pipeline_state solve_tri_pipeline_state(src0->ne[0], src1->ne[0]); - GGML_ASSERT(dst->buffer != nullptr); + vk_pipeline pipeline = nullptr; - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op); - GGML_ASSERT(pipeline != nullptr); + { + std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); + auto it = ctx->device->pipeline_solve_tri_f32.find(solve_tri_pipeline_state); + if (it != ctx->device->pipeline_solve_tri_f32.end()) { + pipeline = it->second; + } else { + ctx->device->pipeline_solve_tri_f32[solve_tri_pipeline_state] = pipeline = std::make_shared<vk_pipeline_struct>(); + } + } - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + return pipeline; + } + return nullptr; + case GGML_OP_ARGMAX: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) { + return ctx->device->pipeline_argmax_f32; + } + return nullptr; + case GGML_OP_COUNT_EQUAL: + if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_I64) { + return ctx->device->pipeline_count_equal_i32; + } + return nullptr; + case GGML_OP_IM2COL: + if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_im2col_f32; + } + if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_im2col_f32_f16; + } + return nullptr; + case GGML_OP_IM2COL_3D: + if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_im2col_3d_f32; + } + if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_im2col_3d_f32_f16; + } + return nullptr; + case GGML_OP_TIMESTEP_EMBEDDING: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_timestep_embedding_f32; + } + return nullptr; + case GGML_OP_CONV_TRANSPOSE_1D: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_conv_transpose_1d_f32; + } + return nullptr; + case GGML_OP_COL2IM_1D: + switch (src0->type) { + case GGML_TYPE_F32: return ctx->device->pipeline_col2im_1d_f32; + case GGML_TYPE_F16: return ctx->device->pipeline_col2im_1d_f16; + case GGML_TYPE_BF16: return ctx->device->pipeline_col2im_1d_bf16; + default: return nullptr; + } + case GGML_OP_POOL_1D: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_pool1d_f32; + } + return nullptr; + case GGML_OP_POOL_2D: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_pool2d_f32; + } + return nullptr; + case GGML_OP_RWKV_WKV6: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_rwkv_wkv6_f32; + } + return nullptr; + case GGML_OP_RWKV_WKV7: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_rwkv_wkv7_f32; + } + return nullptr; + case GGML_OP_GATED_LINEAR_ATTN: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_gated_linear_attn_f32; + } + return nullptr; + case GGML_OP_LIGHTNING_INDEXER: + // only the k type selects a pipeline, the other types are fixed by ggml_lightning_indexer() + if (ggml_vk_lightning_indexer_k_type_supported(src1->type)) { + return ctx->device->pipeline_lightning_indexer_f32[src1->type]; + } + return nullptr; + case GGML_OP_GATED_DELTA_NET: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + const uint32_t S_v = dst->src[2]->ne[0]; + const uint32_t kda = (dst->src[3]->ne[0] == (int64_t)S_v) ? 1 : 0; + uint32_t si; + switch (S_v) { + case 16: si = 0; break; + case 32: si = 1; break; + case 64: si = 2; break; + case 128: si = 3; break; + default: return nullptr; + } + return ctx->device->pipeline_gated_delta_net[si][kda]; + } + return nullptr; + case GGML_OP_SSM_SCAN: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + const uint32_t d_state = src0->ne[0]; + if (d_state == 128) { + return ctx->device->pipeline_ssm_scan_f32_d128; + } else if (d_state == 256) { + return ctx->device->pipeline_ssm_scan_f32_d256; + } + } + return nullptr; + case GGML_OP_SSM_CONV: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + switch (ctx->num_additional_fused_ops) { + case 0: return ctx->device->pipeline_ssm_conv_f32; + case 1: return ctx->device->pipeline_ssm_conv_silu_f32; + case 2: return ctx->device->pipeline_ssm_conv_bias_silu_f32; + default: return nullptr; + } + } + return nullptr; + case GGML_OP_OPT_STEP_ADAMW: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_opt_step_adamw_f32; + } + return nullptr; + case GGML_OP_OPT_STEP_SGD: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_opt_step_sgd_f32; + } + return nullptr; + case GGML_OP_LEAKY_RELU: + if (src0->type == dst->type && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { + return ctx->device->pipeline_leaky_relu[dst->type == GGML_TYPE_F16]; + } + return nullptr; + case GGML_OP_CONV_2D: + case GGML_OP_CONV_TRANSPOSE_2D: + if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + uint32_t K = dst->ne[2]; // Cout + uint32_t NPQ = dst->ne[3] * dst->ne[1] * dst->ne[0]; // N * OH * OW + vk_conv_shapes shape = ggml_vk_conv_select_shape(ctx, K, NPQ); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); - vk_subbuffer src_buf[5] = {}; - for (int i = 0; i < 5; i++) { - src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]); - } + bool transpose = dst->op == GGML_OP_CONV_TRANSPOSE_2D; + uint32_t KW = (uint32_t)src0->ne[0]; + uint32_t KH = (uint32_t)src0->ne[1]; + uint32_t s0 = (uint32_t)(ggml_get_op_params_i32(dst, 0)); + uint32_t s1 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 1) : s0; + uint32_t p0 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 2) : 0; + uint32_t p1 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 3) : 0; + uint32_t d0 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 4) : 1; + uint32_t d1 = !transpose ? (uint32_t)ggml_get_op_params_i32(dst, 5) : 1; - const vk_op_gated_linear_attn_push_constants pc = { - (uint32_t)n_seqs, - (uint32_t)seq_length, - (uint32_t)n_embed, - (uint32_t)n_heads, - scale, - }; + // tile-aligned shapes let the shader skip bounds checks + const uint32_t Cin = (uint32_t)src1->ne[2]; + const uint32_t CRS = Cin * KW * KH; + const uint32_t BS_K = vk_conv_block_sizes[shape].K; + const uint32_t BS_CRS = vk_conv_block_sizes[shape].CRS; + const uint32_t BS_NPQ = vk_conv_block_sizes[shape].NPQ; + const uint32_t aligned = ((K % BS_K == 0) && + (CRS % BS_CRS == 0) && + (NPQ % BS_NPQ == 0)) ? 1u : 0u; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], dst_buf}, - pc, { (uint32_t)(n_seqs * n_heads), 1, 1 }); -} + vk_conv2d_pipeline_state conv2d_pipeline_state(s0, s1, p0, p1, d0, d1, KW, KH, aligned); -static void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - const ggml_tensor * q = dst->src[0]; - const ggml_tensor * k = dst->src[1]; - const ggml_tensor * w = dst->src[2]; - const ggml_tensor * m = dst->src[3]; + std::map<vk_conv2d_pipeline_state, vk_pipeline> *pipelines = nullptr; + if (op == GGML_OP_CONV_2D) { + if (src0->type == GGML_TYPE_F32) { + pipelines = &ctx->device->pipeline_conv2d_f32[shape]; + } else if (src0->type == GGML_TYPE_F16) { + pipelines = &ctx->device->pipeline_conv2d_f16_f32[shape]; + } + } else if (op == GGML_OP_CONV_TRANSPOSE_2D) { + if (src0->type == GGML_TYPE_F32) { + pipelines = &ctx->device->pipeline_conv_transpose_2d_f32[shape]; + } else if (src0->type == GGML_TYPE_F16) { + pipelines = &ctx->device->pipeline_conv_transpose_2d_f16_f32[shape]; + } + } - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, q, k, w, dst, dst->op); - GGML_ASSERT(pipeline != nullptr); + vk_pipeline pipeline = nullptr; - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + { + std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); + auto it = pipelines->find(conv2d_pipeline_state); + if (it != pipelines->end()) { + pipeline = it->second; + } else { + (*pipelines)[conv2d_pipeline_state] = pipeline = std::make_shared<vk_pipeline_struct>(); + } + } - const uint32_t n_kv = k->ne[2]; - const uint32_t n_heads = q->ne[1]; - const uint32_t n_tokens = q->ne[2]; - const uint32_t n_streams = q->ne[3]; - const uint32_t n_masks = m->ne[3]; + return pipeline; + } + return nullptr; + case GGML_OP_CONV_2D_DW: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + if (ggml_is_contiguous(src1)) { + return ctx->device->pipeline_conv2d_dw_whcn_f32; + } else if (ggml_is_contiguous_channels(src1)) { + return ctx->device->pipeline_conv2d_dw_cwhn_f32; + } + } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) { + if (ggml_is_contiguous(src1)) { + return ctx->device->pipeline_conv2d_dw_whcn_f16_f32; + } else if (ggml_is_contiguous_channels(src1)) { + return ctx->device->pipeline_conv2d_dw_cwhn_f16_f32; + } + } + return nullptr; + case GGML_OP_CONV_3D: + if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + const uint32_t OC = (uint32_t)ggml_get_op_params_i32(dst, 11); + const uint32_t IC = (uint32_t)ggml_get_op_params_i32(dst, 9); + const uint32_t N = (uint32_t)ggml_get_op_params_i32(dst, 10); + const uint32_t NPQ = N * dst->ne[2] * dst->ne[1] * dst->ne[0]; + const vk_conv_shapes shape = ggml_vk_conv_select_shape(ctx, OC, NPQ); - const uint32_t n_outputs = (uint32_t)(dst->ne[0] * dst->ne[1] * dst->ne[3]); - const uint32_t dispatch_x = std::min(n_outputs, ctx->device->properties.limits.maxComputeWorkGroupCount[0]); - const uint32_t dispatch_y = CEIL_DIV(n_outputs, dispatch_x); + const uint32_t KW = (uint32_t)src0->ne[0]; + const uint32_t KH = (uint32_t)src0->ne[1]; + const uint32_t KD = (uint32_t)src0->ne[2]; + const uint32_t s0 = (uint32_t)ggml_get_op_params_i32(dst, 0); + const uint32_t s1 = (uint32_t)ggml_get_op_params_i32(dst, 1); + const uint32_t s2 = (uint32_t)ggml_get_op_params_i32(dst, 2); + const uint32_t p0 = (uint32_t)ggml_get_op_params_i32(dst, 3); + const uint32_t p1 = (uint32_t)ggml_get_op_params_i32(dst, 4); + const uint32_t p2 = (uint32_t)ggml_get_op_params_i32(dst, 5); + const uint32_t d0 = (uint32_t)ggml_get_op_params_i32(dst, 6); + const uint32_t d1 = (uint32_t)ggml_get_op_params_i32(dst, 7); + const uint32_t d2 = (uint32_t)ggml_get_op_params_i32(dst, 8); - // q, w and dst are f32 and m is f16, so their strides are passed in elements; - // k may be quantized, so its strides stay in bytes - const uint32_t q_nb1 = q->nb[1] / sizeof(float); - const uint32_t q_nb2 = q->nb[2] / sizeof(float); - const uint32_t q_nb3 = q->nb[3] / sizeof(float); - const uint32_t k_nb2 = k->nb[2]; - const uint32_t k_nb3 = k->nb[3]; - const uint32_t w_nb1 = w->nb[1] / sizeof(float); - const uint32_t w_nb3 = w->nb[3] / sizeof(float); - const uint32_t m_nb1 = m->nb[1] / sizeof(ggml_fp16_t); - const uint32_t m_nb3 = m->nb[3] / sizeof(ggml_fp16_t); - const uint32_t d_nb1 = dst->nb[1] / sizeof(float); - const uint32_t d_nb3 = dst->nb[3] / sizeof(float); + const uint32_t CRS = IC * KW * KH * KD; + const uint32_t BS_K = vk_conv_block_sizes[shape].K; + const uint32_t BS_CRS = vk_conv_block_sizes[shape].CRS; + const uint32_t BS_NPQ = vk_conv_block_sizes[shape].NPQ; + const uint32_t aligned = ((OC % BS_K == 0) && + (CRS % BS_CRS == 0) && + (NPQ % BS_NPQ == 0)) ? 1u : 0u; - const vk_op_lightning_indexer_push_constants pc = { - n_kv, n_heads, n_tokens, n_streams, n_masks, dispatch_x, - q_nb1, q_nb2, q_nb3, - k_nb2, k_nb3, - w_nb1, w_nb3, - m_nb1, m_nb3, - d_nb1, d_nb3, - }; + vk_conv3d_pipeline_state conv3d_pipeline_state(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {ggml_vk_tensor_subbuffer(ctx, q), ggml_vk_tensor_subbuffer(ctx, k), ggml_vk_tensor_subbuffer(ctx, w), ggml_vk_tensor_subbuffer(ctx, m), ggml_vk_tensor_subbuffer(ctx, dst)}, - pc, {dispatch_x, dispatch_y, 1}); -} + std::map<vk_conv3d_pipeline_state, vk_pipeline> *pipelines = nullptr; + if (src0->type == GGML_TYPE_F32) { + pipelines = &ctx->device->pipeline_conv3d_f32[shape]; + } else if (src0->type == GGML_TYPE_F16) { + pipelines = &ctx->device->pipeline_conv3d_f16_f32[shape]; + } else { + return nullptr; + } -static void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - const ggml_tensor * src_q = dst->src[0]; - const ggml_tensor * src_v = dst->src[2]; - const ggml_tensor * src_beta = dst->src[4]; + vk_pipeline pipeline = nullptr; - GGML_ASSERT(dst->buffer != nullptr); + { + std::lock_guard<std::mutex> guard(ctx->device->compile_mutex); + auto it = pipelines->find(conv3d_pipeline_state); + if (it != pipelines->end()) { + pipeline = it->second; + } else { + (*pipelines)[conv3d_pipeline_state] = pipeline = std::make_shared<vk_pipeline_struct>(); + } + } - const uint32_t S_v = (uint32_t)src_v->ne[0]; - const uint32_t H = (uint32_t)src_v->ne[1]; - const uint32_t n_tokens = (uint32_t)src_v->ne[2]; - const uint32_t n_seqs = (uint32_t)src_v->ne[3]; + return pipeline; + } + return nullptr; + case GGML_OP_ADD1: + if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_add1_f16_f16; + } + if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_add1_f16_f32; + } + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_add1_f32_f32; + } + return nullptr; + case GGML_OP_ARANGE: + if (dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_arange_f32; + } + return nullptr; + case GGML_OP_FILL: + if (dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_fill_f32; + } + if (dst->type == GGML_TYPE_F16) { + return ctx->device->pipeline_fill_f16; + } + return nullptr; + default: + return nullptr; + } - // K (snapshot slot count) is an op param; state holds s0 only [S_v, S_v, H, n_seqs]. - const uint32_t K = (uint32_t)ggml_get_op_params_i32(dst, 0); + GGML_UNUSED(src2); +} - const uint32_t s_off = S_v * H * n_tokens * n_seqs; +template<typename PC> +static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst, ggml_op op, PC&& pc, vk_pipeline pipeline_override = nullptr) { + VK_LOG_DEBUG("ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; + if (src1 != nullptr) { + std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; + } + if (src2 != nullptr) { + std::cerr << "), (" << src2 << ", name=" << src2->name << ", type=" << src2->type << ", ne0=" << src2->ne[0] << ", ne1=" << src2->ne[1] << ", ne2=" << src2->ne[2] << ", ne3=" << src2->ne[3] << ", nb0=" << src2->nb[0] << ", nb1=" << src2->nb[1] << ", nb2=" << src2->nb[2] << ", nb3=" << src2->nb[3]; + } + if (src3 != nullptr) { + std::cerr << "), (" << src3 << ", name=" << src3->name << ", type=" << src3->type << ", ne0=" << src3->ne[0] << ", ne1=" << src3->ne[1] << ", ne2=" << src3->ne[2] << ", ne3=" << src3->ne[3] << ", nb0=" << src3->nb[0] << ", nb1=" << src3->nb[1] << ", nb2=" << src3->nb[2] << ", nb3=" << src3->nb[3]; + } + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; + std::cerr << "), " << ggml_op_name(op) << ")"); + GGML_ASSERT(op == GGML_OP_GET_ROWS || op == GGML_OP_CPY || op == GGML_OP_CONCAT || (!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type)))); // NOLINT + GGML_ASSERT(dst->buffer != nullptr); + const uint64_t ne00 = src0->ne[0]; + const uint64_t ne01 = src0->ne[1]; + const uint64_t ne02 = src0->ne[2]; + const uint64_t ne03 = src0->ne[3]; - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op); - GGML_ASSERT(pipeline != nullptr); + const bool use_src1 = src1 != nullptr; + const uint64_t ne10 = use_src1 ? src1->ne[0] : 0; + const uint64_t ne11 = use_src1 ? src1->ne[1] : 0; + const uint64_t ne12 = use_src1 ? src1->ne[2] : 0; + const uint64_t ne13 = use_src1 ? src1->ne[3] : 0; - // Chunked (UT-transform) path for short multi-token blocks with a scalar gate: the - // autoregressive shader serialises one cross-lane reduction per token, which is what - // makes MTP verify blocks cost ~n tokens. KDA, n_tokens == 1 and long blocks keep the - // autoregressive shader. GGML_GDN_CHUNKED=0 forces it off for A/B. - { - static const bool chunked_enabled = []() { - const char * s = getenv("GGML_GDN_CHUNKED"); - return s == nullptr || atoi(s) != 0; - }(); + const bool use_src2 = src2 != nullptr; + const bool use_src3 = src3 != nullptr; - const bool kda = (dst->src[3]->ne[0] == (int64_t) S_v); + init_pushconst_fastdiv(pc); - int si = -1; - switch (S_v) { - case 16: si = 0; break; - case 32: si = 1; break; - case 64: si = 2; break; - case 128: si = 3; break; - default: si = -1; break; - } + vk_pipeline pipeline; + if (pipeline_override) { + pipeline = pipeline_override; + } else { + pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op); + } - if (chunked_enabled && !kda && si >= 0 && - n_tokens >= 2 && n_tokens <= 16 && - ctx->device->gated_delta_net_chunked_cols[si] != 0 && - ctx->device->pipeline_gated_delta_net_chunked[si] != nullptr) { - pipeline = ctx->device->pipeline_gated_delta_net_chunked[si]; + if (pipeline == nullptr) { + std::cerr << "ggml_vulkan: Error: Missing op: " << ggml_op_name(op) << " for " << ggml_type_name(src0->type); + if (src1 != nullptr) { + std::cerr << " and " << ggml_type_name(src1->type); } + std::cerr << " to " << ggml_type_name(dst->type) << std::endl; + GGML_ABORT("fatal error"); } ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); - vk_subbuffer src_buf[6] = {}; - for (int i = 0; i < 6; i++) { - src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]); - } + vk_subbuffer src0_buf = ggml_vk_tensor_subbuffer(ctx, src0, true); + vk_subbuffer src1_buf = use_src1 ? ggml_vk_tensor_subbuffer(ctx, src1, true) : vk_subbuffer{}; + vk_subbuffer src2_buf = use_src2 ? ggml_vk_tensor_subbuffer(ctx, src2, true) : vk_subbuffer{}; + vk_subbuffer src3_buf = use_src3 ? ggml_vk_tensor_subbuffer(ctx, src3, true) : vk_subbuffer{}; + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); - const uint32_t sq1 = (uint32_t)(src_q->nb[1] / sizeof(float)); - const uint32_t sq2 = (uint32_t)(src_q->nb[2] / sizeof(float)); - const uint32_t sq3 = (uint32_t)(src_q->nb[3] / sizeof(float)); - const uint32_t sv1 = (uint32_t)(src_v->nb[1] / sizeof(float)); - const uint32_t sv2 = (uint32_t)(src_v->nb[2] / sizeof(float)); - const uint32_t sv3 = (uint32_t)(src_v->nb[3] / sizeof(float)); - const uint32_t sb1 = (uint32_t)(src_beta->nb[1] / sizeof(float)); - const uint32_t sb2 = (uint32_t)(src_beta->nb[2] / sizeof(float)); - const uint32_t sb3 = (uint32_t)(src_beta->nb[3] / sizeof(float)); + // Compute misalignment offset for descriptors and store it in in push constants. + init_pushconst_tensor_offsets(ctx, pc, src0, src1, src2, src3, dst); - const uint32_t neq1 = (uint32_t)src_q->ne[1]; - const uint32_t rq3 = (uint32_t)(src_v->ne[3] / src_q->ne[3]); + std::array<uint32_t, 3> elements; - const float scale = 1.0f / sqrtf((float)S_v); - const vk_op_gated_delta_net_push_constants pc = { - H, n_tokens, n_seqs, s_off, - sq1, sq2, sq3, - sv1, sv2, sv3, - sb1, sb2, sb3, - neq1, rq3, - scale, - K - }; + switch (op) { + case GGML_OP_NORM: + case GGML_OP_RMS_NORM_BACK: + case GGML_OP_L2_NORM: + case GGML_OP_SOFT_MAX: + case GGML_OP_SOFT_MAX_BACK: + case GGML_OP_SUM_ROWS: + case GGML_OP_CUMSUM: + case GGML_OP_MEAN: + case GGML_OP_ARGMAX: + { + const uint32_t nr = ggml_nrows(src0); + if (nr > 262144) { + elements = { 512, 512, CEIL_DIV(nr, 262144) }; + } else if (nr > 512) { + elements = { 512, CEIL_DIV(nr, 512), 1 }; + } else { + elements = { nr, 1, 1 }; + } + } break; + case GGML_OP_SINKHORN_NORM: + { + // One invocation per TOKEN, not per row: the shader holds the whole + // n*n matrix for a token in registers. ggml_nrows() here would be + // n*nt and dispatch n x more invocations than exist tokens, all of + // which return immediately on the t >= nt guard. + const uint32_t nt = (uint32_t)(ne02 * ne03); + if (nt > 262144) { + elements = { 512, 512, CEIL_DIV(nt, 262144) }; + } else if (nt > 512) { + elements = { 512, CEIL_DIV(nt, 512), 1 }; + } else { + elements = { nt, 1, 1 }; + } + } break; + case GGML_OP_SOLVE_TRI: + { + uint32_t nr = (uint32_t)(ne02 * ne03); + if (nr > 262144) { + elements = { 512, 512, CEIL_DIV(nr, 262144) }; + } else if (nr > 512) { + elements = { 512, CEIL_DIV(nr, 512), 1 }; + } else { + elements = { nr, 1, 1 }; + } + } + break; + case GGML_OP_RMS_NORM: + if (ctx->do_add_rms_partials) { + // Run one element per thread, 128 threads per workgroup + elements = { (uint32_t)CEIL_DIV(ne00, 128), 1, 1 }; + } else { + elements = { (uint32_t)ne01, (uint32_t)ne02, (uint32_t)ne03 }; + } + break; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf}, - pc, { H, n_seqs, S_v }); -} + case GGML_OP_SUM: + // We use GGML_OP_SUM_ROWS with 1 row. + elements = { 1, 1, 1 }; + break; + case GGML_OP_GROUP_NORM: + { + const uint32_t num_groups = dst->op_params[0]; + elements = { num_groups * (uint32_t)src0->ne[3], 1, 1 }; + } break; + case GGML_OP_DIAG_MASK_INF: + elements = { (uint32_t)ggml_nrows(src0), (uint32_t)ne00, 1 }; + break; + case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: + { + uint32_t nrows = (uint32_t)ggml_nrows(src0); + uint32_t z = 1; + if (nrows > ctx->device->properties.limits.maxComputeWorkGroupCount[0]) { + z = CEIL_DIV(nrows, 32768); + nrows = 32768; + } + elements = { nrows, (uint32_t)ne00, z }; -static void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - const ggml_tensor * src2 = dst->src[2]; - const ggml_tensor * src3 = dst->src[3]; - const ggml_tensor * src4 = dst->src[4]; - const ggml_tensor * src5 = dst->src[5]; + } break; + case GGML_OP_GET_ROWS: + elements = { (uint32_t)ne00, (uint32_t)ne10, (uint32_t)(ne11 * ne12) }; + elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); + break; + case GGML_OP_GET_ROWS_BACK: + elements = { (uint32_t)dst->ne[0], (uint32_t)dst->ne[1], 1 }; + elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + break; + case GGML_OP_ARGSORT: + GGML_ASSERT(0); + break; + case GGML_OP_IM2COL: + { + const bool is_2D = dst->op_params[6] == 1; - GGML_ASSERT(dst->buffer != nullptr); + const uint32_t IC = src1->ne[is_2D ? 2 : 1]; - const uint32_t head_dim = src0->ne[1]; - const uint32_t n_head = src1->ne[1]; - const uint32_t n_group = src4->ne[1]; - const uint32_t n_tok = src1->ne[2]; - const uint32_t n_seq = src1->ne[3]; + const uint32_t KH = is_2D ? src0->ne[1] : 1; + const uint32_t KW = src0->ne[0]; - bool is_mamba2 = (src3->nb[1] == sizeof(float)); - GGML_ASSERT(is_mamba2); + const uint32_t OH = is_2D ? dst->ne[2] : 1; + const uint32_t OW = dst->ne[1]; - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, dst->op); - GGML_ASSERT(pipeline != nullptr); + const uint32_t batch = src1->ne[is_2D ? 3 : 2]; - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + const uint32_t CHW = IC * KH * KW; + // Cap X workgroups to limit concurrent IC channel reads. + // The shader loops over X to cover the full CHW dimension. + // AMD prefers a lower limit + const uint32_t min_cap = ctx->device->vendor_id == VK_VENDOR_ID_AMD ? 512u : 4096u; + const uint32_t x_elements = std::min(CHW, std::max(min_cap, OW * KH * KW)); + elements = { x_elements, OW, OH * batch }; + elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); + } break; + case GGML_OP_IM2COL_3D: + { + const uint32_t IC = ((const uint32_t *)(dst->op_params))[9]; - const int64_t s_off = ggml_nelements(src1) * sizeof(float); + const uint32_t N = ne13 / IC; - const vk_op_ssm_scan_push_constants pc = { - (uint32_t)src0->nb[2], (uint32_t)src0->nb[3], - (uint32_t)src1->nb[2], (uint32_t)src1->nb[3], - (uint32_t)src2->nb[1], (uint32_t)src2->nb[2], - (uint32_t)src3->nb[1], - (uint32_t)src4->nb[2], (uint32_t)src4->nb[3], - (uint32_t)src5->nb[2], (uint32_t)src5->nb[3], - (uint32_t)s_off, - n_head, head_dim, n_group, n_tok, - n_seq, (uint32_t) ggml_get_op_params_i32(dst, 0) - }; + const uint32_t KD = ne02; + const uint32_t KH = ne01; + const uint32_t KW = ne00; - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); - vk_subbuffer src_buf[7] = {}; - for (int i = 0; i < 7 && dst->src[i] != nullptr; i++) { - src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]); - } + const uint32_t OD = dst->ne[3] / N; + const uint32_t OH = dst->ne[2]; + const uint32_t OW = dst->ne[1]; - std::array<uint32_t, 3> elements; + const uint32_t IC_KD_KH_KW = IC*KD*KH*KW; + const uint32_t N_OD_OH = N*OD*OH; - const uint32_t d_state = src0->ne[0]; - uint32_t num_subgroups = d_state / ctx->device->subgroup_size; - const uint32_t num_workgroups_x = CEIL_DIV(n_head * head_dim, num_subgroups); - const uint32_t num_workgroups_y = n_seq; - elements = { num_workgroups_x, num_workgroups_y, 1 }; + elements = { IC_KD_KH_KW, OW, N_OD_OH }; + elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); + } break; + case GGML_OP_TIMESTEP_EMBEDDING: + { + const uint32_t dim = dst->op_params[0]; + uint32_t half_ceil = (dim + 1) / 2; + elements = { half_ceil, (uint32_t)src0->ne[0], 1 }; + } break; + case GGML_OP_CONV_TRANSPOSE_1D: + { + elements = {uint32_t(src0->ne[1]), 1, 1}; // parallelize in {Cout, 1, 1} + } break; + case GGML_OP_COL2IM_1D: + { + elements = { uint32_t(dst->ne[0]), uint32_t(dst->ne[1]), 1 }; + } break; + case GGML_OP_POOL_1D: + { + const uint32_t N = dst->ne[3] * dst->ne[2]; + const uint32_t OC = dst->ne[1]; + const uint32_t OL = dst->ne[0]; + elements = { N * OC * OL, 1, 1}; + } break; + case GGML_OP_POOL_2D: + { + const uint32_t N = dst->ne[3]; + const uint32_t OC = dst->ne[2]; + const uint32_t OH = dst->ne[1]; + const uint32_t OW = dst->ne[0]; + elements = { N * OC * OH * OW, 1, 1}; + } break; + case GGML_OP_CONV_2D: + case GGML_OP_CONV_TRANSPOSE_2D: + if constexpr (std::is_same_v<PC, vk_op_conv2d_push_constants>) { + const uint32_t NPQ = pc.N * pc.OH * pc.OW; + const vk_conv_shapes shape = ggml_vk_conv_select_shape(ctx, pc.Cout, NPQ); + const uint32_t NPQ_blocks = CEIL_DIV(NPQ, vk_conv_block_sizes[shape].NPQ); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], src_buf[6], dst_buf}, - pc, elements); -} + elements = { pc.Cout, NPQ_blocks, 1 }; + if (elements[1] > 512) { + elements[2] = CEIL_DIV(elements[1], 512); + elements[1] = 512; + } + } else { + GGML_ABORT("invalid push constant type for CONV_2D"); + } + break; + case GGML_OP_CONV_3D: + if constexpr (std::is_same_v<PC, vk_op_conv3d_push_constants>) { + const uint32_t NPQ = pc.N * pc.OD * pc.OH * pc.OW; + const vk_conv_shapes shape = ggml_vk_conv_select_shape(ctx, pc.OC, NPQ); + const uint32_t NPQ_blocks = CEIL_DIV(NPQ, vk_conv_block_sizes[shape].NPQ); -static void ggml_vk_ssm_conv(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { - ggml_tensor * conv = cgraph->nodes[node_idx]; - const ggml_tensor * src0 = conv->src[0]; - const ggml_tensor * src1 = conv->src[1]; + elements = { pc.OC, NPQ_blocks, 1 }; + if (elements[1] > 512) { + elements[2] = CEIL_DIV(elements[1], 512); + elements[1] = 512; + } + } else { + GGML_ABORT("invalid push constant type for CONV_3D"); + } + break; + case GGML_OP_ADD: + case GGML_OP_SUB: + case GGML_OP_DIV: + case GGML_OP_MUL: + case GGML_OP_ADD1: + case GGML_OP_OUT_PROD: + case GGML_OP_ARANGE: + case GGML_OP_FILL: + case GGML_OP_SCALE: + case GGML_OP_SQR: + case GGML_OP_SQRT: + case GGML_OP_SIN: + case GGML_OP_COS: + case GGML_OP_LOG: + case GGML_OP_TRI: + case GGML_OP_DIAG: + case GGML_OP_CLAMP: + case GGML_OP_LEAKY_RELU: + case GGML_OP_PAD: + case GGML_OP_PAD_REFLECT_1D: + case GGML_OP_ROLL: + case GGML_OP_REPEAT: + case GGML_OP_REPEAT_BACK: + case GGML_OP_CPY: + case GGML_OP_CONCAT: + case GGML_OP_UPSCALE: + case GGML_OP_UNARY: + case GGML_OP_GLU: + case GGML_OP_CONV_2D_DW: + { + uint32_t ne = ggml_nelements(dst); + if (op == GGML_OP_CPY && ggml_is_quantized(src0->type) && ggml_is_quantized(dst->type)) { + // Convert from number of logical elements to 2- or 4-byte units. + ne /= ggml_blck_size(src0->type); + if ((ggml_type_size(src0->type) % 4) == 0) { + ne *= ggml_type_size(src0->type) / 4; + } else { + ne *= ggml_type_size(src0->type) / 2; + } + } + if (op == GGML_OP_CONCAT && ggml_is_quantized(dst->type)) { + ne = ne / ggml_blck_size(dst->type) * ggml_type_size(dst->type) / ggml_vk_concat_unit_size(dst->type); + } + // copy_to_quant has block size of 32, and each thread does QUANT_K elements. + // Splitting into 512x512xZ wouldn't work well since each workgroup does 1024 elements. + // So divide by block size here before splitting into 512x512 groups. + if (op == GGML_OP_CPY && !ggml_is_quantized(src0->type) && ggml_is_quantized(dst->type)) { + ne = CEIL_DIV(ne, ggml_blck_size(dst->type)); + } + if (ne > 262144) { + elements = { 512, 512, CEIL_DIV(ne, 262144) }; + } else if (ne > 512) { + elements = { 512, CEIL_DIV(ne, 512), 1 }; + } else { + elements = { ne, 1, 1 }; + } - // Pick the destination tensor (last node in the fused chain) and the optional bias. - // Fusion modes: 0 = ssm_conv, 1 = ssm_conv+silu, 2 = ssm_conv+add(bias)+silu. - ggml_tensor * dst = conv; - const ggml_tensor * bias = nullptr; + if (pipeline == ctx->device->pipeline_cpy_transpose_02_32 || + pipeline == ctx->device->pipeline_cpy_transpose_02_16) { + // 32x32 tiles over dims 0 and 2; dim1 and dim3 are the batch + elements[0] = (uint32_t)CEIL_DIV(dst->ne[0], 32); + elements[1] = (uint32_t)CEIL_DIV(dst->ne[2], 32); + elements[2] = (uint32_t)(dst->ne[1]*dst->ne[3]); + elements[0] = std::min(elements[0], ctx->device->properties.limits.maxComputeWorkGroupCount[0]); + elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); + } else if (pipeline == ctx->device->pipeline_cpy_transpose_32 || + pipeline == ctx->device->pipeline_cpy_transpose_16) { + // 32x32 tiles + elements[0] = (uint32_t)CEIL_DIV(dst->ne[0], 32); + elements[1] = (uint32_t)CEIL_DIV(dst->ne[1], 32); + elements[2] = (uint32_t)(dst->ne[2]*dst->ne[3]); + elements[0] = std::min(elements[0], ctx->device->properties.limits.maxComputeWorkGroupCount[0]); + elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + elements[2] = std::min(elements[2], ctx->device->properties.limits.maxComputeWorkGroupCount[2]); + } + } break; + case GGML_OP_ADD_ID: + { + elements = { (uint32_t)ne01, (uint32_t)ne02, 1 }; + } break; + case GGML_OP_SET_ROWS: + { + uint32_t ne = ggml_nelements(src0); + if (dst->type == GGML_TYPE_TURBO4_0) { + // turbo4_0 uses a bespoke COOPERATIVE shader: one workgroup (128 threads) + // per QUANT_K=128 block, so dispatch one workgroup per block. + ne = CEIL_DIV(ne, ggml_blck_size(dst->type)); + } else if (ggml_is_quantized(dst->type)) { + // quants run 32 threads each doing QUANT_K elements + ne = CEIL_DIV(ne, 32 * ggml_blck_size(dst->type)); + } else { + // scalar types do one element per thread, running 512 threads + ne = CEIL_DIV(ne, 512); + } + if (ne > 262144) { + elements = { 512, 512, CEIL_DIV(ne, 262144) }; + } else if (ne > 512) { + elements = { 512, CEIL_DIV(ne, 512), 1 }; + } else { + elements = { ne, 1, 1 }; + } + } + break; + case GGML_OP_SSM_CONV: + { + const uint32_t nr = src0->ne[1]; + const uint32_t n_t = dst->ne[1]; + const uint32_t n_s = dst->ne[2]; + elements = { nr, n_t, n_s }; + } + break; + default: + elements = { (uint32_t)ggml_nelements(src0), 1, 1 }; + break; + } - if (ctx->num_additional_fused_ops == 1) { - dst = cgraph->nodes[node_idx + 1]; // silu - } else if (ctx->num_additional_fused_ops == 2) { - ggml_tensor * add = cgraph->nodes[node_idx + 1]; - bias = (add->src[0] == conv) ? add->src[1] : add->src[0]; - dst = cgraph->nodes[node_idx + 2]; // silu + if (op == GGML_OP_ADD || op == GGML_OP_RMS_NORM) { + vk_subbuffer a_buf = src0_buf; + if (ctx->do_add_rms_partials) { + a_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_add_rms_partials, ctx->prealloc_size_add_rms_partials_offset); + } + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { src0_buf, src1_buf, dst_buf, a_buf }, pc, elements); + } else if (op == GGML_OP_GLU) { + // Empty src1 is possible in glu, but the shader needs a buffer + vk_subbuffer subbuf1 = use_src1 ? src1_buf : src0_buf; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, subbuf1, dst_buf }, pc, elements); + } else if (op == GGML_OP_SOFT_MAX) { + // Empty src1 and src2 is possible in soft_max, but the shader needs a buffer + vk_subbuffer subbuf1 = use_src1 ? src1_buf : src0_buf; + vk_subbuffer subbuf2 = use_src2 ? src2_buf : src0_buf; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, subbuf1, subbuf2, dst_buf }, pc, elements); + } else if (op == GGML_OP_ROPE || op == GGML_OP_ROPE_BACK) { + // Empty src2 and src3 is possible in rope, but the shader needs a buffer + vk_subbuffer subbuf2 = use_src2 ? src2_buf : src0_buf; + vk_subbuffer subbuf3 = use_src3 ? src3_buf : src0_buf; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, subbuf2, dst_buf, subbuf3 }, pc, elements); + } else if (op == GGML_OP_IM2COL || op == GGML_OP_IM2COL_3D) { + if (ctx->device->shader_int64 && ctx->device->buffer_device_address) { + // buffer device address path doesn't use dst buffer + dst_buf.size = 1; + } + // im2col uses only src1 and dst buffers + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src1_buf, dst_buf }, pc, elements); + } else if (op == GGML_OP_COUNT_EQUAL) { + // count_equal assumes that destination buffer is initialized with zeroes + ggml_vk_buffer_memset_async(subctx, dst_buf.buffer, dst_buf.offset, 0, dst_buf.size); + ggml_vk_sync_buffers(ctx, subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, dst_buf }, pc, elements); + } else if (op == GGML_OP_OPT_STEP_SGD) { + // OPT_STEP_SGD works on src0, it does not need dst + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, src2_buf }, pc, elements); + } else if (use_src3) { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, src2_buf, src3_buf, dst_buf }, pc, elements); + } else if (use_src2) { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, src2_buf, dst_buf }, pc, elements); + } else if (use_src1) { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, dst_buf }, pc, elements); + } else { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, dst_buf }, pc, elements); } +} - // The shader always declares 4 bindings; bind src0 as a dummy when bias isn't fused. - const ggml_tensor * src2 = bias ? bias : src0; +void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32<vk_op_ssm_conv_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_SSM_CONV, { - (uint32_t)src0->nb[1], (uint32_t)src0->nb[2], - (uint32_t)src1->nb[1], - (uint32_t)dst->nb[0], (uint32_t)dst->nb[1], (uint32_t)dst->nb[2], - (uint32_t)src1->ne[0], - (uint32_t)src0->ne[0], - (uint32_t)src0->ne[1], - (uint32_t)dst->ne[1], - (uint32_t)dst->ne[2], + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_GET_ROWS, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, 0, }); } -static void ggml_vk_op_f32_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, const vk_op_push_constants&& pc) { - const ggml_tensor * x = dst->src[0]; - const ggml_tensor * g = dst->src[1]; - const ggml_tensor * gm = dst->src[2]; - const ggml_tensor * gv = dst->src[3]; - const ggml_tensor * p = dst->src[4]; - - GGML_ASSERT(x->type == GGML_TYPE_F32); - GGML_ASSERT(g->type == GGML_TYPE_F32); - GGML_ASSERT(gm->type == GGML_TYPE_F32); - GGML_ASSERT(gv->type == GGML_TYPE_F32); - GGML_ASSERT(p->type == GGML_TYPE_F32); - GGML_ASSERT(dst->buffer != nullptr); - GGML_ASSERT(ggml_is_contiguous(x)); - GGML_ASSERT(ggml_is_contiguous(g)); - GGML_ASSERT(ggml_is_contiguous(gm)); - GGML_ASSERT(ggml_is_contiguous(gv)); - GGML_ASSERT(ggml_is_contiguous(p)); - GGML_ASSERT(ggml_are_same_shape(x, g)); - GGML_ASSERT(ggml_are_same_shape(x, gm)); - GGML_ASSERT(ggml_are_same_shape(x, gv)); - GGML_ASSERT(ggml_nelements(p) == 7); - - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, g, gm, gv, dst, GGML_OP_OPT_STEP_ADAMW); - GGML_ASSERT(pipeline != nullptr); +void ggml_vk_get_rows_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_GET_ROWS_BACK, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2], (uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, 0, + }); +} - vk_subbuffer x_buf = ggml_vk_tensor_subbuffer(ctx, x); - vk_subbuffer g_buf = ggml_vk_tensor_subbuffer(ctx, g); - vk_subbuffer gm_buf = ggml_vk_tensor_subbuffer(ctx, gm); - vk_subbuffer gv_buf = ggml_vk_tensor_subbuffer(ctx, gv); - vk_subbuffer p_buf = ggml_vk_tensor_subbuffer(ctx, p); +void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); - std::array<uint32_t, 3> elements = { (uint32_t)ggml_nelements(x), 1, 1 }; + int nb1 = dst->op_params[0] / src0_type_size; // 4 bytes of float32 + int nb2 = dst->op_params[1] / src0_type_size; // 4 bytes of float32 + int nb3 = dst->op_params[2] / src0_type_size; // 4 bytes of float32 + int offset = dst->op_params[3] / src0_type_size; // offset in bytes - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {x_buf, g_buf, gm_buf, gv_buf, p_buf}, - pc, elements); + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, dst->op, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t)nb1, (uint32_t)nb2, (uint32_t)nb3, + 0, + 0.0f, 0.0f, offset, + }); } -static void ggml_vk_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - const size_t n = ggml_nelements(dst->src[0]); - - ggml_vk_op_f32_opt_step_adamw( - ctx, subctx, dst, - { (uint32_t)n, 0, 0.0f, 0.0f, 0.0f, 0.0f } - ); -} +void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) { + const ggml_tensor *first_node = cgraph->nodes[node_idx]; + const ggml_tensor *dst = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; -static void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { - const size_t n = ggml_nelements(dst->src[0]); + // Make a list of all the tensors used by the op. + // Last element of the list is the dest tensor. + const ggml_tensor *tensors[MAX_PARAMETER_COUNT]; + uint32_t num_srcs = ctx->num_additional_fused_ops + 2; + uint32_t num_tensors = num_srcs + 1; + GGML_ASSERT(num_tensors + ctx->do_add_rms_partials <= MAX_PARAMETER_COUNT); - ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_OPT_STEP_SGD, { (uint32_t)n, 0, 0.0f, 0.0f, 0.0f, 0.0f }); -} + tensors[0] = first_node->src[0]; + tensors[1] = first_node->src[1]; + for (int32_t i = 0; i < ctx->num_additional_fused_ops; ++i) { + // check whether the previous result is src[0] or src[1] + if (cgraph->nodes[node_idx + i] == cgraph->nodes[node_idx + i + 1]->src[0]) { + tensors[i+2] = cgraph->nodes[node_idx + i + 1]->src[1]; + } else { + tensors[i+2] = cgraph->nodes[node_idx + i + 1]->src[0]; + } + } + tensors[num_srcs] = dst; -static void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - int * op_params = (int *)dst->op_params; + vk_op_multi_add_push_constants pc; + pc.ne20 = (uint32_t)dst->ne[0]; + pc.ne21 = (uint32_t)dst->ne[1]; + pc.ne22 = (uint32_t)dst->ne[2]; + pc.ne23 = (uint32_t)dst->ne[3]; - const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type); - const uint32_t units_per_block = ggml_type_size(dst->type) / unit_size; - const uint32_t block_size = ggml_blck_size(dst->type); - const bool quantized = ggml_is_quantized(dst->type); + for (uint32_t i = 0; i < num_tensors; ++i) { + const ggml_tensor *t = tensors[i]; + pc.nb[i][0] = (uint32_t)t->nb[0] / sizeof(float); + pc.nb[i][1] = (uint32_t)t->nb[1] / sizeof(float); + pc.nb[i][2] = (uint32_t)t->nb[2] / sizeof(float); + pc.nb[i][3] = (uint32_t)t->nb[3] / sizeof(float); + } + pc.rms_partials = ctx->do_add_rms_partials; - // Address dimension 0 in packed storage units; higher strides may be noncontiguous. - const uint32_t ne00 = src0->ne[0] / block_size * units_per_block; - const uint32_t ne10 = src1->ne[0] / block_size * units_per_block; - const uint32_t ne20 = dst->ne[0] / block_size * units_per_block; - const uint32_t nb00 = quantized ? 1 : src0->nb[0] / unit_size; - const uint32_t nb10 = quantized ? 1 : src1->nb[0] / unit_size; - const uint32_t nb20 = quantized ? 1 : dst->nb[0] / unit_size; + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, tensors[0], tensors[1], nullptr, dst, dst->op); - vk_op_concat_push_constants pc {{ - ne20 * (uint32_t)dst->ne[1] * (uint32_t)dst->ne[2] * (uint32_t)dst->ne[3], - ne00, (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], nb00, (uint32_t)src0->nb[1] / unit_size, (uint32_t)src0->nb[2] / unit_size, (uint32_t)src0->nb[3] / unit_size, - ne10, (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], nb10, (uint32_t)src1->nb[1] / unit_size, (uint32_t)src1->nb[2] / unit_size, (uint32_t)src1->nb[3] / unit_size, - ne20, (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], nb20, (uint32_t) dst->nb[1] / unit_size, (uint32_t) dst->nb[2] / unit_size, (uint32_t) dst->nb[3] / unit_size, - 0, - 0.0f, 0.0f, op_params[0], - }}; - ggml_vk_op_f32<vk_op_concat_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONCAT, std::move(pc)); -} + if (pipeline == nullptr) { + std::cerr << "ggml_vulkan: Error: Missing multi_add"; + GGML_ABORT("fatal error"); + } -static void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t mode = (uint32_t)ggml_get_op_params_i32(dst, 0); + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - GGML_TENSOR_UNARY_OP_LOCALS + ggml_backend_vk_buffer_context * buf_ctx[MAX_PARAMETER_COUNT]; + vk_buffer buf[MAX_PARAMETER_COUNT]; + size_t offset[MAX_PARAMETER_COUNT]; + bool uma[MAX_PARAMETER_COUNT]; - float sf0 = (float)ne0 / ne00; - float sf1 = (float)ne1 / ne01; - float sf2 = (float)ne2 / ne02; - float sf3 = (float)ne3 / ne03; - float pixel_offset = 0.5f; + for (uint32_t i = 0; i < num_tensors; ++i) { + buf_ctx[i] = (ggml_backend_vk_buffer_context *)tensors[i]->buffer->context; + buf[i] = nullptr; + offset[i] = 0; + uma[i] = false; - if (mode & GGML_SCALE_FLAG_ALIGN_CORNERS) { - sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0; - sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1; - pixel_offset = 0.0f; + if (ctx->device->uma) { + ggml_vk_host_get(ctx->device, tensors[i]->data, buf[i], offset[i]); + uma[i] = buf[i] != nullptr; + } + if (!uma[i]) { + buf[i] = buf_ctx[i]->dev_buffer; + offset[i] = vk_tensor_offset(tensors[i]) + tensors[i]->view_offs; + } + GGML_ASSERT(buf[i] != nullptr); + } + // If any remaining descriptors are unused, just point them at src[0] + for (uint32_t i = num_tensors; i < MAX_PARAMETER_COUNT; ++i) { + buf[i] = buf[0]; + offset[i] = 0; + } + if (ctx->do_add_rms_partials) { + buf[num_tensors] = ctx->prealloc_add_rms_partials; + offset[num_tensors] = ctx->prealloc_size_add_rms_partials_offset; } - ggml_vk_op_f32<vk_op_upscale_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UPSCALE, { - (uint32_t)ggml_nelements(dst), 0, 0, - (uint32_t)ne00, (uint32_t)ne01, - (uint32_t)nb00 / src0_type_size, (uint32_t)nb01 / src0_type_size, (uint32_t)nb02 / src0_type_size, (uint32_t)nb03 / src0_type_size, - (uint32_t)ne0, (uint32_t)ne1, (uint32_t)ne2, (uint32_t)ne3, - sf0, sf1, sf2, sf3, pixel_offset - }); -} - -static void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); - p.param1 = ggml_get_op_params_f32(dst, 0); - p.param2 = ggml_get_op_params_f32(dst, 1); - - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SCALE, std::move(p)); -} + std::array<uint32_t, 3> elements; -static void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SQR, vk_op_unary_push_constants_init(src0, dst)); -} + uint32_t ne = ggml_nelements(dst); + if (ne > 262144) { + elements = { 512, 512, CEIL_DIV(ne, 262144) }; + } else if (ne > 512) { + elements = { 512, CEIL_DIV(ne, 512), 1 }; + } else { + elements = { ne, 1, 1 }; + } -static void ggml_vk_sqrt(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SQRT, vk_op_unary_push_constants_init(src0, dst)); + static_assert(MAX_PARAMETER_COUNT == 12); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + ggml_vk_subbuffer(ctx, buf[0], offset[0]), + ggml_vk_subbuffer(ctx, buf[1], offset[1]), + ggml_vk_subbuffer(ctx, buf[2], offset[2]), + ggml_vk_subbuffer(ctx, buf[3], offset[3]), + ggml_vk_subbuffer(ctx, buf[4], offset[4]), + ggml_vk_subbuffer(ctx, buf[5], offset[5]), + ggml_vk_subbuffer(ctx, buf[6], offset[6]), + ggml_vk_subbuffer(ctx, buf[7], offset[7]), + ggml_vk_subbuffer(ctx, buf[8], offset[8]), + ggml_vk_subbuffer(ctx, buf[9], offset[9]), + ggml_vk_subbuffer(ctx, buf[10], offset[10]), + ggml_vk_subbuffer(ctx, buf[11], offset[11]), + }, pc, elements); } -static void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t src1_type_size = ggml_type_size(src1->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ADD1, { + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ADD, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, 0, - 0.0f, 0.0f, 0, + 0.0f, 0.0f, ctx->do_add_rms_partials, }); } -static void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - VK_LOG_DEBUG("ggml_vk_arange(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")"); +void ggml_vk_out_prod(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); - vk_op_push_constants pc = { + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_OUT_PROD, { (uint32_t)ggml_nelements(dst), - 1, - ggml_get_op_params_f32(dst, 0), - ggml_get_op_params_f32(dst, 2), - 0.0f, 0.0f, - }; - - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, dst, GGML_OP_ARANGE); - GGML_ASSERT(pipeline != nullptr); - - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, false); - - std::array<uint32_t, 3> elements = { (uint32_t)ggml_nelements(dst), 1, 1 }; - - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements); + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], + (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], + (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], + (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, 0, + }); } -static void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - VK_LOG_DEBUG("ggml_vk_fill(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")"); - - vk_op_push_constants pc = { - (uint32_t)ggml_nelements(dst), - 1, - ggml_get_op_params_f32(dst, 0), - 0.0f, - 0.0f, 0.0f, - }; - - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, dst, GGML_OP_FILL); - GGML_ASSERT(pipeline != nullptr); - - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, false); - - std::array<uint32_t, 3> elements = { (uint32_t)ggml_nelements(dst), 1, 1 }; +void ggml_vk_sub(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements); + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SUB, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, 0, + }); } -static void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SIN, vk_op_unary_push_constants_init(src0, dst)); -} +void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); -static void ggml_vk_cos(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_COS, vk_op_unary_push_constants_init(src0, dst)); + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_MUL, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, 0, + }); } -static void ggml_vk_log(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LOG, vk_op_unary_push_constants_init(src0, dst)); +int ggml_vk_unary_mul_op_index(ggml_unary_op op) { + switch (op) { + case GGML_UNARY_OP_GELU: return 0; + case GGML_UNARY_OP_SIGMOID: return 1; + case GGML_UNARY_OP_SILU: return 2; + case GGML_UNARY_OP_SOFTPLUS: return 3; + default: return -1; + } } -static void ggml_vk_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); - p.param1 = ggml_get_op_params_f32(dst, 0); - - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_TRI, std::move(p)); -} +void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { + const ggml_tensor * unary = cgraph->nodes[node_idx]; + ggml_tensor * mul = cgraph->nodes[node_idx + 1]; -static void ggml_vk_diag(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst)); + // unary on src1 that tiles into src0 + const bool op_on_b = mul->src[1] == unary && + !ggml_are_same_shape(unary->src[0], mul->src[0]) && + ggml_can_repeat(unary, mul->src[0]); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_DIAG, std::move(p)); -} + const ggml_tensor * src0 = op_on_b ? mul->src[0] : unary->src[0]; + const ggml_tensor * src1 = op_on_b ? unary->src[0] : + ((mul->src[0] == unary) ? mul->src[1] : mul->src[0]); -static void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); - p.param1 = ggml_get_op_params_f32(dst, 0); - p.param2 = ggml_get_op_params_f32(dst, 1); + const bool f16 = src0->type == GGML_TYPE_F16; + const bool norepeat = ggml_are_same_shape(src0, src1); + const int oi = ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)); + if (oi < 0) { + GGML_ABORT("fatal error"); + } + vk_pipeline pipeline = ctx->device->pipeline_unary_mul[oi][f16][norepeat][op_on_b]; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CLAMP, std::move(p)); -} + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(mul->type); -static void ggml_vk_pad(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - vk_op_pad_push_constants p = vk_op_pad_push_constants_init(src0, dst); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_PAD, std::move(p)); + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, mul, GGML_OP_UNARY, { + (uint32_t)ggml_nelements(op_on_b ? mul : src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) mul->ne[0], (uint32_t) mul->ne[1], (uint32_t) mul->ne[2],(uint32_t) mul->ne[3], (uint32_t) mul->nb[0] / dst_type_size, (uint32_t) mul->nb[1] / dst_type_size, (uint32_t) mul->nb[2] / dst_type_size, (uint32_t) mul->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, 0, + }, pipeline); } -static void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - const uint32_t p0 = (uint32_t)dst->op_params[0]; - const uint32_t p1 = (uint32_t)dst->op_params[1]; - - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst)); - memcpy(&p.param1, &p0, sizeof(float)); - memcpy(&p.param2, &p1, sizeof(float)); +void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_PAD_REFLECT_1D, std::move(p)); + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_DIV, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, 0, + }); } -static void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - const int32_t s0 = ggml_get_op_params_i32(dst, 0); - const int32_t s1 = ggml_get_op_params_i32(dst, 1); - const int32_t s2 = ggml_get_op_params_i32(dst, 2); - const int32_t s3 = ggml_get_op_params_i32(dst, 3); - const uint32_t s01_packed = ((s0 + 0x8000) << 16) | (s1 + 0x8000); - const uint32_t s23_packed = ((s2 + 0x8000) << 16) | (s3 + 0x8000); - - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); - memcpy(&p.param1, &s01_packed, sizeof(float)); - memcpy(&p.param2, &s23_packed, sizeof(float)); - - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_ROLL, std::move(p)); -} +void ggml_vk_add_id(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t src2_type_size = ggml_type_size(src2->type); -static void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst)); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_REPEAT, std::move(p)); + ggml_vk_op_f32<vk_op_add_id_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_ADD_ID, { + (uint32_t)dst->ne[0], + (uint32_t)dst->ne[1], + (uint32_t)src0->nb[1] / src0_type_size, + (uint32_t)src0->nb[2] / src0_type_size, + (uint32_t)src1->nb[1] / src1_type_size, + (uint32_t)src2->nb[1] / src2_type_size, + }); } -static void ggml_vk_repeat_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst)); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_REPEAT_BACK, std::move(p)); -} +static void ggml_vk_op_f32_wkv(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, const vk_op_rwkv_wkv6_push_constants&& pc, int version) { + GGML_ASSERT(version == 6 || version == 7); + int num_srcs = version == 6 ? 6 : 7; -static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - // Contiguous same-type device copies: vkCmdCopyBuffer. Consecutive copies - // that share src/dst buffers become one command with many regions. - if (src0->type == dst->type && - ggml_is_contiguous(src0) && ggml_is_contiguous(dst) && - src0->buffer != nullptr && dst->buffer != nullptr && - subctx != nullptr && subctx->s != nullptr) { - vk_buffer src_buf = nullptr; - vk_buffer dst_buf = nullptr; - size_t src_off = 0; - size_t dst_off = 0; - if (ctx->device->uma) { - ggml_vk_host_get(ctx->device, src0->data, src_buf, src_off); - ggml_vk_host_get(ctx->device, dst->data, dst_buf, dst_off); - } - if (src_buf == nullptr) { - auto * src_ctx = (ggml_backend_vk_buffer_context *) src0->buffer->context; - src_buf = src_ctx->dev_buffer; - src_off = vk_tensor_offset(src0) + src0->view_offs; - } - if (dst_buf == nullptr) { - auto * dst_ctx = (ggml_backend_vk_buffer_context *) dst->buffer->context; - dst_buf = dst_ctx->dev_buffer; - dst_off = vk_tensor_offset(dst) + dst->view_offs; - } - const size_t nbytes = ggml_nbytes(src0); - if (src_buf != nullptr && dst_buf != nullptr && src_buf != dst_buf && - nbytes > 0 && nbytes == ggml_nbytes(dst) && - src_off + nbytes <= src_buf->size && - dst_off + nbytes <= dst_buf->size) { - if (subctx->cpy_src != src_buf || subctx->cpy_dst != dst_buf) { - ggml_vk_flush_cpy_regions(subctx); - subctx->cpy_src = src_buf; - subctx->cpy_dst = dst_buf; - } - subctx->cpy_regions.push_back(vk::BufferCopy{ - (vk::DeviceSize) src_off, (vk::DeviceSize) dst_off, (vk::DeviceSize) nbytes }); - return; - } + for (int i = 0; i < num_srcs; i++) { + GGML_ASSERT(!ggml_is_quantized(dst->src[i]->type)); } - ggml_vk_flush_cpy_regions(subctx); + GGML_ASSERT(dst->buffer != nullptr); - uint32_t ne = (uint32_t)ggml_nelements(src0); - if (ggml_is_quantized(src0->type) && ggml_is_quantized(dst->type)) { - // Convert from number of logical elements to 2- or 4-byte units. - ne /= ggml_blck_size(src0->type); - if ((ggml_type_size(src0->type) % 4) == 0) { - ne *= ggml_type_size(src0->type) / 4; - } else { - ne *= ggml_type_size(src0->type) / 2; - } - } + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op); + GGML_ASSERT(pipeline != nullptr); - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ne); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CPY, std::move(p)); -} + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); + vk_subbuffer src_buf[7] = {}; + for (int i = 0; i < num_srcs; i++) { + src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]); + } -static void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); + std::array<uint32_t, 3> elements = { + (uint32_t)(pc.B * pc.H), + 1, + 1 + }; - // Skip empty skip_rows operations. For most ops the empty check at the start - // of ggml_vk_build_graph is sufficient, but set_rows can have a nonempty dst - // with empty srcs. - if (ggml_is_empty(src0) || ggml_is_empty(src1)) { - return; + if (version == 6) { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf}, + pc, elements); + } else if (version == 7) { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], src_buf[6], dst_buf}, + pc, elements); + } else { + // shouldn't happen + GGML_ASSERT(false); } - - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SET_ROWS, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - 0.0f, 0.0f, 0, - }); } -static void ggml_vk_silu_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SILU_BACK, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f, 0.0f, 0.0f }); +void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const size_t seq_length = dst->src[0]->ne[2]; + const size_t n_embed = dst->ne[0]; + const size_t n_heads = dst->src[0]->ne[1]; + const size_t n_seqs = dst->src[5]->ne[1]; + + ggml_vk_op_f32_wkv( + ctx, subctx, dst, + { + (uint32_t)n_seqs, + (uint32_t)seq_length, + (uint32_t)n_embed, + (uint32_t)n_heads, + }, + 6 + ); } -static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - float * op_params = (float *)dst->op_params; - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); - p.param1 = op_params[0]; +void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const size_t seq_length = dst->src[0]->ne[2]; + const size_t n_embed = dst->ne[0]; + const size_t n_heads = dst->src[0]->ne[1]; + const size_t n_seqs = dst->src[6]->ne[1]; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_NORM, std::move(p)); + ggml_vk_op_f32_wkv( + ctx, subctx, dst, + { + (uint32_t)n_seqs, + (uint32_t)seq_length, + (uint32_t)n_embed, + (uint32_t)n_heads, + }, + 7 + ); } -static void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - const int * int_op_params = (const int *)dst->op_params; - const float * float_op_params = (const float *)dst->op_params; +void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const size_t seq_length = dst->src[0]->ne[2]; + const size_t n_embed = dst->ne[0]; + const size_t n_heads = dst->src[0]->ne[1]; + const size_t n_seqs = dst->src[4]->ne[1]; - const uint32_t num_groups = int_op_params[0]; - const float eps = float_op_params[1]; - const uint32_t group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups); + float scale; + memcpy(&scale, dst->op_params, sizeof(float)); - ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_GROUP_NORM, { group_size, 0, eps, 0.0f, 0.0f, 0.0f }); -} + GGML_ASSERT(dst->buffer != nullptr); -static uint32_t ggml_vk_rms_num_partials(ggml_backend_vk_context * ctx, const ggml_tensor *node) { - const uint32_t ne = (uint32_t)node->ne[0]; - const uint32_t denom = ctx->device->pipeline_add_rms[0][0][0]->wg_denoms[0]; - const uint32_t num_partials = CEIL_DIV(ne, denom); - return num_partials; -} + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op); + GGML_ASSERT(pipeline != nullptr); -static uint32_t ggml_vk_rms_partials_size(ggml_backend_vk_context * ctx, const ggml_tensor *node) { - const uint32_t num_partials = ggml_vk_rms_num_partials(ctx, node); - const uint32_t num_bytes = ROUNDUP_POW2(num_partials * sizeof(uint32_t), ctx->device->partials_binding_alignment); - return num_bytes; -} + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); -static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor *dst, const ggml_tensor *src0, const bool has_ff, bool backprop, const uint32_t set_rows_stride) { - const int n_dims = ((const int32_t *) dst->op_params)[1]; - const int mode = ((const int32_t *) dst->op_params)[2]; - const int n_offs = ((const int32_t *) dst->op_params)[15]; - // const int n_ctx = ((const int32_t *) dst->op_params)[3]; - const int n_ctx_orig = ((const int32_t *) dst->op_params)[4]; - const float freq_base = ((const float *) dst->op_params)[5]; - const float freq_scale = ((const float *) dst->op_params)[6]; - const float ext_factor = ((const float *) dst->op_params)[7]; - const float attn_factor = ((const float *) dst->op_params)[8]; - const float beta_fast = ((const float *) dst->op_params)[9]; - const float beta_slow = ((const float *) dst->op_params)[10]; - int sections[4] {}; - if (mode & GGML_ROPE_TYPE_MROPE) { - memcpy(sections, (const int32_t *) dst->op_params + 11, sizeof(int)*4); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); + vk_subbuffer src_buf[5] = {}; + for (int i = 0; i < 5; i++) { + src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]); } - const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; + const vk_op_gated_linear_attn_push_constants pc = { + (uint32_t)n_seqs, + (uint32_t)seq_length, + (uint32_t)n_embed, + (uint32_t)n_heads, + scale, + }; - float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], dst_buf}, + pc, { (uint32_t)(n_seqs * n_heads), 1, 1 }); +} - const float theta_scale = powf(freq_base, -2.0f/n_dims); +void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k = dst->src[1]; + const ggml_tensor * w = dst->src[2]; + const ggml_tensor * m = dst->src[3]; - uint32_t nb01 = src0->nb[1] / ggml_type_size(src0->type); - uint32_t nb02 = src0->nb[2] / ggml_type_size(src0->type); - uint32_t nb03 = src0->nb[3] / ggml_type_size(src0->type); + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, q, k, w, dst, dst->op); + GGML_ASSERT(pipeline != nullptr); - uint32_t nb11 = dst->nb[1] / ggml_type_size(dst->type); - uint32_t nb12 = dst->nb[2] / ggml_type_size(dst->type); - uint32_t nb13 = dst->nb[3] / ggml_type_size(dst->type); + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - vk_op_rope_push_constants rope { - (uint32_t)mode, (uint32_t)ggml_nrows(src0), (uint32_t)n_dims, (uint32_t)n_offs, freq_scale, - freq_base, ext_factor, attn_factor, {corr_dims[0], corr_dims[1]}, theta_scale, has_ff, - { sections[0], sections[1], sections[2], sections[3] }, is_imrope, backprop, set_rows_stride, + const uint32_t n_kv = k->ne[2]; + const uint32_t n_heads = q->ne[1]; + const uint32_t n_tokens = q->ne[2]; + const uint32_t n_streams = q->ne[3]; + const uint32_t n_masks = m->ne[3]; - (uint32_t)src0->ne[0], - (uint32_t)src0->ne[1], - (uint32_t)src0->ne[2], - nb01, nb02, nb03, - nb11, nb12, nb13, - 0, 0, // a_offset, d_offset filled in by init_pushconst_tensor_offsets + const uint32_t n_outputs = (uint32_t)(dst->ne[0] * dst->ne[1] * dst->ne[3]); + const uint32_t dispatch_x = std::min(n_outputs, ctx->device->properties.limits.maxComputeWorkGroupCount[0]); + const uint32_t dispatch_y = CEIL_DIV(n_outputs, dispatch_x); + + // q, w and dst are f32 and m is f16, so their strides are passed in elements; + // k may be quantized, so its strides stay in bytes + const uint32_t q_nb1 = q->nb[1] / sizeof(float); + const uint32_t q_nb2 = q->nb[2] / sizeof(float); + const uint32_t q_nb3 = q->nb[3] / sizeof(float); + const uint32_t k_nb2 = k->nb[2]; + const uint32_t k_nb3 = k->nb[3]; + const uint32_t w_nb1 = w->nb[1] / sizeof(float); + const uint32_t w_nb3 = w->nb[3] / sizeof(float); + const uint32_t m_nb1 = m->nb[1] / sizeof(ggml_fp16_t); + const uint32_t m_nb3 = m->nb[3] / sizeof(ggml_fp16_t); + const uint32_t d_nb1 = dst->nb[1] / sizeof(float); + const uint32_t d_nb3 = dst->nb[3] / sizeof(float); + + const vk_op_lightning_indexer_push_constants pc = { + n_kv, n_heads, n_tokens, n_streams, n_masks, dispatch_x, + q_nb1, q_nb2, q_nb3, + k_nb2, k_nb3, + w_nb1, w_nb3, + m_nb1, m_nb3, + d_nb1, d_nb3, }; - return rope; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {ggml_vk_tensor_subbuffer(ctx, q), ggml_vk_tensor_subbuffer(ctx, k), ggml_vk_tensor_subbuffer(ctx, w), ggml_vk_tensor_subbuffer(ctx, m), ggml_vk_tensor_subbuffer(ctx, dst)}, + pc, {dispatch_x, dispatch_y, 1}); } -static void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, float * op_params) { - ggml_tensor * dst; - const ggml_tensor * src0; - const ggml_tensor * src1; +void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const ggml_tensor * src_q = dst->src[0]; + const ggml_tensor * src_v = dst->src[2]; + const ggml_tensor * src_beta = dst->src[4]; - if (ctx->num_additional_fused_ops > 0) { - // fused rms_norm + mul - ggml_tensor *mul = cgraph->nodes[node_idx + 1]; - ggml_tensor *other_src = mul->src[0] == cgraph->nodes[node_idx + 0] ? mul->src[1] : mul->src[0]; - dst = mul; - src0 = cgraph->nodes[node_idx]->src[0]; - src1 = other_src; - } else { - dst = cgraph->nodes[node_idx]; - src0 = src1 = dst->src[0]; - } + GGML_ASSERT(dst->buffer != nullptr); - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); + const uint32_t S_v = (uint32_t)src_v->ne[0]; + const uint32_t H = (uint32_t)src_v->ne[1]; + const uint32_t n_tokens = (uint32_t)src_v->ne[2]; + const uint32_t n_seqs = (uint32_t)src_v->ne[3]; - uint32_t param3 = ctx->do_add_rms_partials ? ggml_vk_rms_num_partials(ctx, dst) : 0; + // K (snapshot slot count) is an op param; state holds s0 only [S_v, S_v, H, n_seqs]. + const uint32_t K = (uint32_t)ggml_get_op_params_i32(dst, 0); - vk_op_binary_push_constants bin { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - op_params[0], 0.0f, (int32_t)param3, - }; + const uint32_t s_off = S_v * H * n_tokens * n_seqs; - // more than one fused op means rms_norm+mul+rope - if (ctx->num_additional_fused_ops > 1) { - static constexpr uint32_t max_tensors = 7; - const ggml_tensor *tensors[max_tensors] {}; + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op); + GGML_ASSERT(pipeline != nullptr); - ggml_tensor *rms = cgraph->nodes[node_idx + 0]; - ggml_tensor *mul = cgraph->nodes[node_idx + 1]; - ggml_tensor *rope = cgraph->nodes[node_idx + 2]; + // Chunked (UT-transform) path for short multi-token blocks with a scalar gate: the + // autoregressive shader serialises one cross-lane reduction per token, which is what + // makes MTP verify blocks cost ~n tokens. KDA, n_tokens == 1 and long blocks keep the + // autoregressive shader. GGML_GDN_CHUNKED=0 forces it off for A/B. + { + static const bool chunked_enabled = []() { + const char * s = getenv("GGML_GDN_CHUNKED"); + return s == nullptr || atoi(s) != 0; + }(); - ggml_tensor *other_src = mul->src[0] == rms ? mul->src[1] : mul->src[0]; + const bool kda = (dst->src[3]->ne[0] == (int64_t) S_v); - bool do_set_rows = ctx->num_additional_fused_ops == 4; + int si = -1; + switch (S_v) { + case 16: si = 0; break; + case 32: si = 1; break; + case 64: si = 2; break; + case 128: si = 3; break; + default: si = -1; break; + } - tensors[0] = rms->src[0]; - tensors[1] = other_src; - tensors[2] = mul; - tensors[3] = rope->src[1]; // pos - tensors[4] = rope->src[2]; // ff - tensors[5] = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; // dst - tensors[6] = do_set_rows ? tensors[5]->src[1] : nullptr; - const uint32_t set_rows_stride = do_set_rows ? tensors[5]->nb[1] / ggml_type_size(tensors[5]->type) : 0; + if (chunked_enabled && !kda && si >= 0 && + n_tokens >= 2 && n_tokens <= 16 && + ctx->device->gated_delta_net_chunked_cols[si] != 0 && + ctx->device->pipeline_gated_delta_net_chunked[si] != nullptr) { + pipeline = ctx->device->pipeline_gated_delta_net_chunked[si]; + } + } + + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - vk_op_rms_norm_mul_rope_push_constants pc; - pc.bin = bin; - pc.rope = ggml_vk_make_rope_constants(rope, rope->src[0], tensors[4] != nullptr, false, set_rows_stride); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); + vk_subbuffer src_buf[6] = {}; + for (int i = 0; i < 6; i++) { + src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]); + } - vk_pipeline pipeline = tensors[5]->type == GGML_TYPE_F16 ? ctx->device->pipeline_rms_norm_mul_rope_f32_f16 : ctx->device->pipeline_rms_norm_mul_rope_f32_f32; + const uint32_t sq1 = (uint32_t)(src_q->nb[1] / sizeof(float)); + const uint32_t sq2 = (uint32_t)(src_q->nb[2] / sizeof(float)); + const uint32_t sq3 = (uint32_t)(src_q->nb[3] / sizeof(float)); + const uint32_t sv1 = (uint32_t)(src_v->nb[1] / sizeof(float)); + const uint32_t sv2 = (uint32_t)(src_v->nb[2] / sizeof(float)); + const uint32_t sv3 = (uint32_t)(src_v->nb[3] / sizeof(float)); + const uint32_t sb1 = (uint32_t)(src_beta->nb[1] / sizeof(float)); + const uint32_t sb2 = (uint32_t)(src_beta->nb[2] / sizeof(float)); + const uint32_t sb3 = (uint32_t)(src_beta->nb[3] / sizeof(float)); - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + const uint32_t neq1 = (uint32_t)src_q->ne[1]; + const uint32_t rq3 = (uint32_t)(src_v->ne[3] / src_q->ne[3]); - ggml_backend_vk_buffer_context * buf_ctx[max_tensors]; - vk_buffer buf[max_tensors]; - size_t offset[max_tensors]; - bool uma[max_tensors]; + const float scale = 1.0f / sqrtf((float)S_v); + const vk_op_gated_delta_net_push_constants pc = { + H, n_tokens, n_seqs, s_off, + sq1, sq2, sq3, + sv1, sv2, sv3, + sb1, sb2, sb3, + neq1, rq3, + scale, + K + }; - for (uint32_t i = 0; i < max_tensors; ++i) { - if (!tensors[i]) { - // If any remaining descriptors are unused, just point them at src[0] - buf[i] = buf[0]; - offset[i] = 0; - continue; - } - buf_ctx[i] = (ggml_backend_vk_buffer_context *)tensors[i]->buffer->context; - buf[i] = nullptr; - offset[i] = 0; - uma[i] = false; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf}, + pc, { H, n_seqs, S_v }); +} - if (ctx->device->uma) { - ggml_vk_host_get(ctx->device, tensors[i]->data, buf[i], offset[i]); - uma[i] = buf[i] != nullptr; - } - if (!uma[i]) { - buf[i] = buf_ctx[i]->dev_buffer; - offset[i] = vk_tensor_offset(tensors[i]) + tensors[i]->view_offs; - } - GGML_ASSERT(buf[i] != nullptr); - } +void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * src2 = dst->src[2]; + const ggml_tensor * src3 = dst->src[3]; + const ggml_tensor * src4 = dst->src[4]; + const ggml_tensor * src5 = dst->src[5]; - // a_offset is unused (the fused path reads from shared memory), but the rope/set_rows dst can be misaligned. - // Round the binding offset down to the storage buffer alignment; the in-element shift goes in pc.rope.d_offset. - pc.rope.d_offset = get_misalign_bytes(ctx, tensors[5]) / ggml_type_size(tensors[5]->type); - offset[5] &= ~(size_t(ctx->device->properties.limits.minStorageBufferOffsetAlignment) - 1); + GGML_ASSERT(dst->buffer != nullptr); - std::array<uint32_t, 3> elements; - elements = { (uint32_t)rms->src[0]->ne[1], (uint32_t)rms->src[0]->ne[2], (uint32_t)rms->src[0]->ne[3] }; + const uint32_t head_dim = src0->ne[1]; + const uint32_t n_head = src1->ne[1]; + const uint32_t n_group = src4->ne[1]; + const uint32_t n_tok = src1->ne[2]; + const uint32_t n_seq = src1->ne[3]; - static_assert(max_tensors == 7); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - { - ggml_vk_subbuffer(ctx, buf[0], offset[0]), - ggml_vk_subbuffer(ctx, buf[1], offset[1]), - ggml_vk_subbuffer(ctx, buf[2], offset[2]), - ggml_vk_subbuffer(ctx, buf[3], offset[3]), - ggml_vk_subbuffer(ctx, buf[4], offset[4]), - ggml_vk_subbuffer(ctx, buf[5], offset[5]), - ggml_vk_subbuffer(ctx, buf[6], offset[6]), - }, pc, elements); - } else { - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_RMS_NORM, std::move(bin)); - } + bool is_mamba2 = (src3->nb[1] == sizeof(float)); + GGML_ASSERT(is_mamba2); - if (ctx->do_add_rms_partials_offset_calculation) { - ctx->prealloc_size_add_rms_partials_offset += ggml_vk_rms_partials_size(ctx, src0); - ctx->do_add_rms_partials = false; - ctx->do_add_rms_partials_offset_calculation = false; - } -} + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, dst->op); + GGML_ASSERT(pipeline != nullptr); -static void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - float * op_params = (float *)dst->op_params; - ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_RMS_NORM_BACK, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f, 0.0f, 0.0f }); -} + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); -static void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - const float * op_params = (const float *)dst->op_params; - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); - p.param1 = op_params[0]; - ggml_vk_op_f32<vk_op_unary_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_L2_NORM, std::move(p)); -} + const int64_t s_off = ggml_nelements(src1) * sizeof(float); -static void ggml_vk_sinkhorn_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - float eps; - memcpy(&eps, dst->op_params, sizeof(eps)); - const int32_t iters = ggml_get_op_params_i32(dst, 1); - const int64_t n = src0->ne[0]; + const vk_op_ssm_scan_push_constants pc = { + (uint32_t)src0->nb[2], (uint32_t)src0->nb[3], + (uint32_t)src1->nb[2], (uint32_t)src1->nb[3], + (uint32_t)src2->nb[1], (uint32_t)src2->nb[2], + (uint32_t)src3->nb[1], + (uint32_t)src4->nb[2], (uint32_t)src4->nb[3], + (uint32_t)src5->nb[2], (uint32_t)src5->nb[3], + (uint32_t)s_off, + n_head, head_dim, n_group, n_tok, + n_seq, (uint32_t) ggml_get_op_params_i32(dst, 0) + }; - GGML_ASSERT(eps >= 0.0f); - GGML_ASSERT(iters >= 1); - GGML_ASSERT(ggml_is_contiguous(src0)); - GGML_ASSERT(ggml_is_contiguous(dst)); - GGML_ASSERT(ggml_are_same_shape(src0, dst)); - GGML_ASSERT(n == src0->ne[1]); - GGML_ASSERT(n == 2 || n == 4 || n == 8); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); + vk_subbuffer src_buf[7] = {}; + for (int i = 0; i < 7 && dst->src[i] != nullptr; i++) { + src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]); + } - ggml_vk_op_f32<vk_op_sinkhorn_norm_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SINKHORN_NORM, { - (uint32_t) n, - (uint32_t) (src0->ne[2] * src0->ne[3]), - eps, - iters, - 0, - 0, - }); -} + std::array<uint32_t, 3> elements; -static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, vk_op_unary_push_constants_init(src0, dst)); -} + const uint32_t d_state = src0->ne[0]; + uint32_t num_subgroups = d_state / ctx->device->subgroup_size; + const uint32_t num_workgroups_x = CEIL_DIV(n_head * head_dim, num_subgroups); + const uint32_t num_workgroups_y = n_seq; + elements = { num_workgroups_x, num_workgroups_y, 1 }; -static void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - float * op_params = (float *)dst->op_params; - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); - p.param1 = op_params[1]; - p.param2 = op_params[2]; - p.param3 = op_params[3]; - p.param4 = op_params[4]; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, std::move(p)); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], src_buf[6], dst_buf}, + pc, elements); } -static void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const float * op_params_f = (const float *)dst->op_params; +void ggml_vk_ssm_conv(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { + ggml_tensor * conv = cgraph->nodes[node_idx]; + const ggml_tensor * src0 = conv->src[0]; + const ggml_tensor * src1 = conv->src[1]; - const bool swapped = (bool)dst->op_params[1]; - const bool split = src1 != nullptr; - const float alpha = op_params_f[2]; - const float limit = op_params_f[3]; + // Pick the destination tensor (last node in the fused chain) and the optional bias. + // Fusion modes: 0 = ssm_conv, 1 = ssm_conv+silu, 2 = ssm_conv+add(bias)+silu. + ggml_tensor * dst = conv; + const ggml_tensor * bias = nullptr; - if (!split) { - GGML_ASSERT(src0->ne[0] / 2 == dst->ne[0]); - } else { - GGML_ASSERT(src0->ne[0] == src1->ne[0]); - GGML_ASSERT(src0->ne[0] == dst->ne[0]); - GGML_ASSERT(src0->type == src1->type); + if (ctx->num_additional_fused_ops == 1) { + dst = cgraph->nodes[node_idx + 1]; // silu + } else if (ctx->num_additional_fused_ops == 2) { + ggml_tensor * add = cgraph->nodes[node_idx + 1]; + bias = (add->src[0] == conv) ? add->src[1] : add->src[0]; + dst = cgraph->nodes[node_idx + 2]; // silu } - const uint32_t mode = split ? 2 : (swapped ? 1 : 0); - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = split ? ggml_type_size(src1->type) : src0_type_size; - const uint32_t dst_type_size = ggml_type_size(dst->type); + // The shader always declares 4 bindings; bind src0 as a dummy when bias isn't fused. + const ggml_tensor * src2 = bias ? bias : src0; - ggml_vk_op_f32<vk_op_glu_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_GLU, - { - (uint32_t)ggml_nelements(dst), - (uint32_t)src0->ne[0], - (uint32_t)dst->ne[0], - mode, - alpha, - limit, - (uint32_t)(src0->nb[0] / src0_type_size), - (uint32_t)(src0->nb[1] / src0_type_size), - (uint32_t)(src0->nb[2] / src0_type_size), - (uint32_t)(src0->nb[3] / src0_type_size), - (uint32_t)((split ? src1->nb[0] : src0->nb[0]) / src1_type_size), - (uint32_t)((split ? src1->nb[1] : src0->nb[1]) / src1_type_size), - (uint32_t)((split ? src1->nb[2] : src0->nb[2]) / src1_type_size), - (uint32_t)((split ? src1->nb[3] : src0->nb[3]) / src1_type_size), - (uint32_t)(dst->nb[0] / dst_type_size), - (uint32_t)(dst->nb[1] / dst_type_size), - (uint32_t)(dst->nb[2] / dst_type_size), - (uint32_t)(dst->nb[3] / dst_type_size), - (uint32_t)dst->ne[1], - (uint32_t)dst->ne[2], - 0, - 0, 0, 0, 0, 0, 0, - }); + ggml_vk_op_f32<vk_op_ssm_conv_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_SSM_CONV, { + (uint32_t)src0->nb[1], (uint32_t)src0->nb[2], + (uint32_t)src1->nb[1], + (uint32_t)dst->nb[0], (uint32_t)dst->nb[1], (uint32_t)dst->nb[2], + (uint32_t)src1->ne[0], + (uint32_t)src0->ne[0], + (uint32_t)src0->ne[1], + (uint32_t)dst->ne[1], + (uint32_t)dst->ne[2], + }); } -static void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - int32_t * op_params = (int32_t *)dst->op_params; - ggml_vk_op_f32<vk_op_diag_mask_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); -} +static void ggml_vk_op_f32_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, const vk_op_push_constants&& pc) { + const ggml_tensor * x = dst->src[0]; + const ggml_tensor * g = dst->src[1]; + const ggml_tensor * gm = dst->src[2]; + const ggml_tensor * gv = dst->src[3]; + const ggml_tensor * p = dst->src[4]; + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(g->type == GGML_TYPE_F32); + GGML_ASSERT(gm->type == GGML_TYPE_F32); + GGML_ASSERT(gv->type == GGML_TYPE_F32); + GGML_ASSERT(p->type == GGML_TYPE_F32); + GGML_ASSERT(dst->buffer != nullptr); + GGML_ASSERT(ggml_is_contiguous(x)); + GGML_ASSERT(ggml_is_contiguous(g)); + GGML_ASSERT(ggml_is_contiguous(gm)); + GGML_ASSERT(ggml_is_contiguous(gv)); + GGML_ASSERT(ggml_is_contiguous(p)); + GGML_ASSERT(ggml_are_same_shape(x, g)); + GGML_ASSERT(ggml_are_same_shape(x, gm)); + GGML_ASSERT(ggml_are_same_shape(x, gv)); + GGML_ASSERT(ggml_nelements(p) == 7); -static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { - float * op_params = (float *)dst->op_params; + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, g, gm, gv, dst, GGML_OP_OPT_STEP_ADAMW); + GGML_ASSERT(pipeline != nullptr); - float scale = op_params[0]; - float max_bias = op_params[1]; + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - const uint32_t ncols = (uint32_t)src0->ne[0]; - const uint32_t nrows_x = (uint32_t)ggml_nrows(src0); - const uint32_t nrows_y = (uint32_t)src0->ne[1]; + vk_subbuffer x_buf = ggml_vk_tensor_subbuffer(ctx, x); + vk_subbuffer g_buf = ggml_vk_tensor_subbuffer(ctx, g); + vk_subbuffer gm_buf = ggml_vk_tensor_subbuffer(ctx, gm); + vk_subbuffer gv_buf = ggml_vk_tensor_subbuffer(ctx, gv); + vk_subbuffer p_buf = ggml_vk_tensor_subbuffer(ctx, p); - const uint32_t ne12 = src1 ? (uint32_t)(src1->ne[2]) : 0u; - const uint32_t ne13 = src1 ? (uint32_t)(src1->ne[3]) : 0u; - const uint32_t nb11 = src1 ? (uint32_t)(src1->nb[1] / src1->nb[0]) : 0u; - const uint32_t nb12 = src1 ? (uint32_t)(src1->nb[2] / src1->nb[0]) : 0u; - const uint32_t nb13 = src1 ? (uint32_t)(src1->nb[3] / src1->nb[0]) : 0u; + std::array<uint32_t, 3> elements = { (uint32_t)ggml_nelements(x), 1, 1 }; - const uint32_t n_head_kv = src0->ne[2]; - const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv)); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {x_buf, g_buf, gm_buf, gv_buf, p_buf}, + pc, elements); +} - const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); +void ggml_vk_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const size_t n = ggml_nelements(dst->src[0]); - vk_op_soft_max_push_constants pc { - ncols, - src1 != nullptr ? nrows_y : (uint32_t)0, - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], - ne12, ne13, - nb11, nb12, nb13, - scale, max_bias, - m0, m1, - n_head_log2, - nrows_x, - src2 != nullptr - }; + ggml_vk_op_f32_opt_step_adamw( + ctx, subctx, dst, + { (uint32_t)n, 0, 0.0f, 0.0f, 0.0f, 0.0f } + ); +} - if (ncols <= 16384) { - ggml_vk_op_f32<vk_op_soft_max_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_SOFT_MAX, std::move(pc)); - } else { +void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { + const size_t n = ggml_nelements(dst->src[0]); - vk_subbuffer buf_a = ggml_vk_tensor_subbuffer(ctx, src0); - vk_subbuffer buf_b = src1 ? ggml_vk_tensor_subbuffer(ctx, src1) : buf_a; - vk_subbuffer buf_c = src2 ? ggml_vk_tensor_subbuffer(ctx, src2) : buf_a; - vk_subbuffer buf_d = ggml_vk_tensor_subbuffer(ctx, dst); + ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_OPT_STEP_SGD, { (uint32_t)n, 0, 0.0f, 0.0f, 0.0f, 0.0f }); +} - uint32_t elems_per_wg = 128 * 4; - uint32_t num_wgs = CEIL_DIV(ncols, elems_per_wg); - size_t tmp_size = num_wgs * nrows_x * sizeof(float); +void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + int * op_params = (int *)dst->op_params; - if (ctx->prealloc_size_x < tmp_size) { - ctx->prealloc_size_x = tmp_size; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if (ctx->prealloc_size_y < tmp_size) { - ctx->prealloc_size_y = tmp_size; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if (ctx->prealloc_x_need_sync || ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } + const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type); + const uint32_t units_per_block = ggml_type_size(dst->type) / unit_size; + const uint32_t block_size = ggml_blck_size(dst->type); + const bool quantized = ggml_is_quantized(dst->type); - vk_subbuffer buf_x = { ctx->prealloc_x, 0, tmp_size }; - vk_subbuffer buf_y = { ctx->prealloc_y, 0, tmp_size }; + // Address dimension 0 in packed storage units; higher strides may be noncontiguous. + const uint32_t ne00 = src0->ne[0] / block_size * units_per_block; + const uint32_t ne10 = src1->ne[0] / block_size * units_per_block; + const uint32_t ne20 = dst->ne[0] / block_size * units_per_block; + const uint32_t nb00 = quantized ? 1 : src0->nb[0] / unit_size; + const uint32_t nb10 = quantized ? 1 : src1->nb[0] / unit_size; + const uint32_t nb20 = quantized ? 1 : dst->nb[0] / unit_size; - std::array<uint32_t, 3> elements = { num_wgs, nrows_x, 1 }; + vk_op_concat_push_constants pc {{ + ne20 * (uint32_t)dst->ne[1] * (uint32_t)dst->ne[2] * (uint32_t)dst->ne[3], + ne00, (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], nb00, (uint32_t)src0->nb[1] / unit_size, (uint32_t)src0->nb[2] / unit_size, (uint32_t)src0->nb[3] / unit_size, + ne10, (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], nb10, (uint32_t)src1->nb[1] / unit_size, (uint32_t)src1->nb[2] / unit_size, (uint32_t)src1->nb[3] / unit_size, + ne20, (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], nb20, (uint32_t) dst->nb[1] / unit_size, (uint32_t) dst->nb[2] / unit_size, (uint32_t) dst->nb[3] / unit_size, + 0, + 0.0f, 0.0f, op_params[0], + }}; + ggml_vk_op_f32<vk_op_concat_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONCAT, std::move(pc)); +} - vk_pipeline pipeline1 = src1 && src1->type == GGML_TYPE_F16 ? ctx->device->pipeline_soft_max_large1_f32_f16 : ctx->device->pipeline_soft_max_large1_f32; - vk_pipeline pipeline2 = src1 && src1->type == GGML_TYPE_F16 ? ctx->device->pipeline_soft_max_large2_f32_f16 : ctx->device->pipeline_soft_max_large2_f32; - vk_pipeline pipeline3 = src1 && src1->type == GGML_TYPE_F16 ? ctx->device->pipeline_soft_max_large3_f32_f16 : ctx->device->pipeline_soft_max_large3_f32; +void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t mode = (uint32_t)ggml_get_op_params_i32(dst, 0); - ggml_pipeline_request_descriptor_sets(ctx, pipeline1, 1); - ggml_pipeline_request_descriptor_sets(ctx, pipeline2, 1); - ggml_pipeline_request_descriptor_sets(ctx, pipeline3, 1); + GGML_TENSOR_UNARY_OP_LOCALS - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline1, { buf_a, buf_b, buf_c, buf_d, buf_x, buf_y }, pc, elements); - ggml_vk_sync_buffers(ctx, subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline2, { buf_a, buf_b, buf_c, buf_d, buf_x, buf_y }, pc, elements); - ggml_vk_sync_buffers(ctx, subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline3, { buf_a, buf_b, buf_c, buf_d, buf_x, buf_y }, pc, elements); + float sf0 = (float)ne0 / ne00; + float sf1 = (float)ne1 / ne01; + float sf2 = (float)ne2 / ne02; + float sf3 = (float)ne3 / ne03; + float pixel_offset = 0.5f; - ctx->prealloc_x_need_sync = true; - ctx->prealloc_y_need_sync = true; + if (mode & GGML_SCALE_FLAG_ALIGN_CORNERS) { + sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0; + sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1; + pixel_offset = 0.0f; } + + ggml_vk_op_f32<vk_op_upscale_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UPSCALE, { + (uint32_t)ggml_nelements(dst), 0, 0, + (uint32_t)ne00, (uint32_t)ne01, + (uint32_t)nb00 / src0_type_size, (uint32_t)nb01 / src0_type_size, (uint32_t)nb02 / src0_type_size, (uint32_t)nb03 / src0_type_size, + (uint32_t)ne0, (uint32_t)ne1, (uint32_t)ne2, (uint32_t)ne3, + sf0, sf1, sf2, sf3, pixel_offset + }); } -static void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - float * op_params = (float *)dst->op_params; - ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SOFT_MAX_BACK, { (uint32_t)src0->ne[0], (uint32_t)ggml_nrows(src0), op_params[0], op_params[1], 0.0f, 0.0f }); +void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); + p.param1 = ggml_get_op_params_f32(dst, 0); + p.param2 = ggml_get_op_params_f32(dst, 1); + + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SCALE, std::move(p)); } -static void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) { - topk_moe_mode mode = ctx->fused_topk_moe_mode; - const bool has_bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS || mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS; - ggml_tensor * logits = cgraph->nodes[node_idx + 0]->src[0]; - ggml_tensor * bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS ? cgraph->nodes[node_idx + 2]->src[1] : - mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? cgraph->nodes[node_idx + 3]->src[1] : - logits; - ggml_tensor * weights = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; - ggml_tensor * ids = mode == TOPK_MOE_SIGMOID_NORM_BIAS ? cgraph->nodes[node_idx + 4] : - mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? cgraph->nodes[node_idx + 5] : - mode == TOPK_MOE_LATE_SOFTMAX ? cgraph->nodes[node_idx + 1] : - cgraph->nodes[node_idx + 3]; +void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SQR, vk_op_unary_push_constants_init(src0, dst)); +} - GGML_ASSERT(logits->type == GGML_TYPE_F32); - GGML_ASSERT(bias->type == GGML_TYPE_F32); - GGML_ASSERT(weights->type == GGML_TYPE_F32); - GGML_ASSERT(ids->type == GGML_TYPE_I32); +void ggml_vk_sqrt(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SQRT, vk_op_unary_push_constants_init(src0, dst)); +} - const int n_experts = logits->ne[0]; - const int n_rows = logits->ne[1]; - const int n_expert_used = weights->ne[1]; +void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); - GGML_ASSERT(ids->nb[1] / ggml_type_size(ids->type) == (size_t) n_experts); + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ADD1, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, 0, + }); +} - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, cgraph->nodes[node_idx], GGML_OP_SOFT_MAX); +void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + VK_LOG_DEBUG("ggml_vk_arange(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")"); + + vk_op_push_constants pc = { + (uint32_t)ggml_nelements(dst), + 1, + ggml_get_op_params_f32(dst, 0), + ggml_get_op_params_f32(dst, 2), + 0.0f, 0.0f, + }; + + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, dst, GGML_OP_ARANGE); + GGML_ASSERT(pipeline != nullptr); ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, false); - vk_subbuffer logits_buf = ggml_vk_tensor_subbuffer(ctx, logits); - vk_subbuffer bias_buf = ggml_vk_tensor_subbuffer(ctx, bias); - vk_subbuffer weights_buf = ggml_vk_tensor_subbuffer(ctx, weights); - vk_subbuffer ids_buf = ggml_vk_tensor_subbuffer(ctx, ids); + std::array<uint32_t, 3> elements = { (uint32_t)ggml_nelements(dst), 1, 1 }; - vk_op_topk_moe_push_constants pc {}; - pc.n_rows = n_rows; - pc.n_experts_push = n_experts; - pc.n_expert_used = n_expert_used; - pc.clamp_min = -std::numeric_limits<float>::infinity(); - pc.clamp_max = std::numeric_limits<float>::infinity(); - if (mode == TOPK_MOE_EARLY_SOFTMAX_NORM) { - ggml_tensor * clamp = cgraph->nodes[node_idx + 7]; - GGML_ASSERT(clamp->op == GGML_OP_CLAMP); - pc.clamp_min = ggml_get_op_params_f32(clamp, 0); - pc.clamp_max = ggml_get_op_params_f32(clamp, 1); - } - if (mode == TOPK_MOE_SIGMOID_NORM_BIAS) { - ggml_tensor * clamp = cgraph->nodes[node_idx + 8]; - GGML_ASSERT(clamp->op == GGML_OP_CLAMP); - pc.clamp_min = ggml_get_op_params_f32(clamp, 0); - pc.clamp_max = ggml_get_op_params_f32(clamp, 1); - } - if (mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS) { - ggml_tensor * clamp = cgraph->nodes[node_idx + 9]; - GGML_ASSERT(clamp->op == GGML_OP_CLAMP); - pc.clamp_min = ggml_get_op_params_f32(clamp, 0); - pc.clamp_max = ggml_get_op_params_f32(clamp, 1); - } + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements); +} -#define GATING_FUNC_SOFTMAX 0 -#define GATING_FUNC_SIGMOID 1 -#define GATING_FUNC_SOFTMAX_WEIGHT 2 -#define GATING_FUNC_SQRT_SOFTPLUS 3 +void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + VK_LOG_DEBUG("ggml_vk_fill(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")"); + const uint64_t n = ggml_nelements(dst); + GGML_ASSERT(n > 0); - pc.gating_func = mode == TOPK_MOE_SIGMOID_NORM_BIAS ? GATING_FUNC_SIGMOID : - mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? GATING_FUNC_SQRT_SOFTPLUS : - mode == TOPK_MOE_LATE_SOFTMAX ? GATING_FUNC_SOFTMAX_WEIGHT : - GATING_FUNC_SOFTMAX; - pc.has_bias = has_bias; - pc.with_norm = mode == TOPK_MOE_EARLY_SOFTMAX_NORM || has_bias; - if (ctx->fused_topk_moe_scale) { - GGML_ASSERT(weights->op == GGML_OP_SCALE); - pc.output_scale = ggml_get_op_params_f32(weights, 0); - pc.output_bias = ggml_get_op_params_f32(weights, 1); - } else { - pc.output_scale = 1.0f; - pc.output_bias = 0.0f; - } + vk_op_push_constants pc = { + (uint32_t)n, + 1, + ggml_get_op_params_f32(dst, 0), + 0.0f, + 0.0f, 0.0f, + }; + + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, dst, GGML_OP_FILL); + GGML_ASSERT(pipeline != nullptr); + + // Split the task distribution to 2D to avoid exceeding maxComputeWorkGroupCount + const uint32_t total_wg = CEIL_DIV(n, pipeline->wg_denoms[0]); + const uint32_t wg_x = std::min(total_wg, ctx->device->properties.limits.maxComputeWorkGroupCount[0]); + const uint32_t wg_y = CEIL_DIV(total_wg, wg_x); + GGML_ASSERT(wg_y <= ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, false); + + std::array<uint32_t, 3> elements = { wg_x * pipeline->wg_denoms[0], wg_y * pipeline->wg_denoms[1], 1 }; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements); +} + +void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SIN, vk_op_unary_push_constants_init(src0, dst)); +} + +void ggml_vk_cos(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_COS, vk_op_unary_push_constants_init(src0, dst)); +} - GGML_ASSERT(n_expert_used <= n_experts); +void ggml_vk_log(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LOG, vk_op_unary_push_constants_init(src0, dst)); +} - const uint32_t rows_per_block = 4; - std::array<uint32_t, 3> elements = { CEIL_DIV(n_rows, rows_per_block), 1, 1 }; +void ggml_vk_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); + p.param1 = ggml_get_op_params_f32(dst, 0); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, {logits_buf, bias_buf, weights_buf, ids_buf}, pc, elements); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_TRI, std::move(p)); } -static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx, bool backprop) { - ggml_tensor * dst = cgraph->nodes[node_idx]; - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - const ggml_tensor * src2 = dst->src[2]; - const ggml_tensor * src3 = nullptr; - const int n_dims = ((int32_t *) dst->op_params)[1]; - const int mode = ((int32_t *) dst->op_params)[2]; - // const int n_ctx = ((int32_t *) dst->op_params)[3]; - const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; - const float freq_base = ((float *) dst->op_params)[5]; - const float beta_fast = ((float *) dst->op_params)[9]; - const float beta_slow = ((float *) dst->op_params)[10]; - int sections[4] {}; - if (mode & GGML_ROPE_TYPE_MROPE) { - memcpy(sections, (int32_t *) dst->op_params + 11, sizeof(int)*4); - } +void ggml_vk_diag(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst)); - float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_DIAG, std::move(p)); +} - uint32_t set_rows_stride = 0; - // Fused rope + view + set_rows passes the set_rows destination stride in set_rows_stride - // and overrides the dst and sets src3=row_indices - if (ctx->num_additional_fused_ops > 0) { - set_rows_stride = cgraph->nodes[node_idx + 2]->nb[1] / ggml_type_size(cgraph->nodes[node_idx + 2]->type); - src3 = cgraph->nodes[node_idx + 2]->src[1]; - dst = cgraph->nodes[node_idx + 2]; - } +void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); + p.param1 = ggml_get_op_params_f32(dst, 0); + p.param2 = ggml_get_op_params_f32(dst, 1); - ggml_vk_op_f32<vk_op_rope_push_constants>(ctx, subctx, src0, src1, src2, src3, dst, GGML_OP_ROPE, - ggml_vk_make_rope_constants(cgraph->nodes[node_idx], src0, src2 != nullptr, backprop, set_rows_stride)); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CLAMP, std::move(p)); } -static void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - const uint32_t * op_params = (const uint32_t *)dst->op_params; +void ggml_vk_pad(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_pad_push_constants p = vk_op_pad_push_constants_init(src0, dst); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_PAD, std::move(p)); +} - uint32_t ncols = src0->ne[0]; - uint32_t nrows = ggml_nrows(src0); +void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + const uint32_t p0 = (uint32_t)dst->op_params[0]; + const uint32_t p1 = (uint32_t)dst->op_params[1]; - uint32_t ncols_pad_log2 = (uint32_t)ceilf(log2f(float(ncols))); - uint32_t ncolsp2 = 1 << ncols_pad_log2; + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst)); + memcpy(&p.param1, &p0, sizeof(float)); + memcpy(&p.param2, &p1, sizeof(float)); - vk_op_argsort_push_constants pc { ncols, ncolsp2, ncols_pad_log2, nrows, op_params[0], 0, 0, 0, 0, }; + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_PAD_REFLECT_1D, std::move(p)); +} - // Pick the largest workgroup size <= ncolsp2 - uint32_t pipeline_idx = std::min(ncols_pad_log2, num_argsort_pipelines - 1); +void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + const int32_t s0 = ggml_get_op_params_i32(dst, 0); + const int32_t s1 = ggml_get_op_params_i32(dst, 1); + const int32_t s2 = ggml_get_op_params_i32(dst, 2); + const int32_t s3 = ggml_get_op_params_i32(dst, 3); + const uint32_t s01_packed = ((s0 + 0x8000) << 16) | (s1 + 0x8000); + const uint32_t s23_packed = ((s2 + 0x8000) << 16) | (s3 + 0x8000); - // Use the "small" argsort shader if the whole sort can be done by a single workgroup. - bool use_small = ncols_pad_log2 <= ctx->device->max_workgroup_size_log2 && - ctx->device->pipeline_argsort_f32[pipeline_idx] != nullptr; + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); + memcpy(&p.param1, &s01_packed, sizeof(float)); + memcpy(&p.param2, &s23_packed, sizeof(float)); - vk_pipeline pipeline = use_small ? ctx->device->pipeline_argsort_f32[pipeline_idx] - : ctx->device->pipeline_argsort_large_f32[pipeline_idx]; + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_ROLL, std::move(p)); +} - vk_subbuffer src0_buf = ggml_vk_tensor_subbuffer(ctx, src0); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); - vk_subbuffer subbuf1 = dst_buf; +void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst)); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_REPEAT, std::move(p)); +} - // Reserve space for ivec2 per element, with rows padded to a power of two - if (!use_small) { - const size_t x_sz = size_t{ncolsp2} * nrows * 2 * sizeof(int); +void ggml_vk_repeat_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst)); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_REPEAT_BACK, std::move(p)); +} - if (ctx->prealloc_size_x < x_sz) { - ctx->prealloc_size_x = x_sz; - ggml_vk_preallocate_buffers(ctx, subctx); +void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + // Contiguous same-type device copies: vkCmdCopyBuffer. Consecutive copies + // that share src/dst buffers become one command with many regions. + if (src0->type == dst->type && + ggml_is_contiguous(src0) && ggml_is_contiguous(dst) && + src0->buffer != nullptr && dst->buffer != nullptr && + subctx != nullptr && subctx->s != nullptr) { + vk_buffer src_buf = nullptr; + vk_buffer dst_buf = nullptr; + size_t src_off = 0; + size_t dst_off = 0; + if (ctx->device->uma) { + ggml_vk_host_get(ctx->device, src0->data, src_buf, src_off); + ggml_vk_host_get(ctx->device, dst->data, dst_buf, dst_off); } - if (ctx->prealloc_x_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); + if (src_buf == nullptr) { + auto * src_ctx = (ggml_backend_vk_buffer_context *) src0->buffer->context; + src_buf = src_ctx->dev_buffer; + src_off = vk_tensor_offset(src0) + src0->view_offs; + } + if (dst_buf == nullptr) { + auto * dst_ctx = (ggml_backend_vk_buffer_context *) dst->buffer->context; + dst_buf = dst_ctx->dev_buffer; + dst_off = vk_tensor_offset(dst) + dst->view_offs; + } + const size_t nbytes = ggml_nbytes(src0); + if (src_buf != nullptr && dst_buf != nullptr && src_buf != dst_buf && + nbytes > 0 && nbytes == ggml_nbytes(dst) && + src_off + nbytes <= src_buf->size && + dst_off + nbytes <= dst_buf->size) { + if (subctx->cpy_src != src_buf || subctx->cpy_dst != dst_buf) { + ggml_vk_flush_cpy_regions(subctx); + subctx->cpy_src = src_buf; + subctx->cpy_dst = dst_buf; + } + subctx->cpy_regions.push_back(vk::BufferCopy{ + (vk::DeviceSize) src_off, (vk::DeviceSize) dst_off, (vk::DeviceSize) nbytes }); + return; } - subbuf1 = { ctx->prealloc_x, 0, ctx->prealloc_x->size }; } - std::array<uint32_t, 3> elements; - - elements[0] = ncolsp2; - elements[1] = std::min((uint32_t)ggml_nrows(src0), ctx->device->properties.limits.maxComputeWorkGroupCount[1]); - elements[2] = 1; + ggml_vk_flush_cpy_regions(subctx); - // First dispatch initializes tmp_idx and does the first N passes where - // there is only communication between threads in the same workgroup. - { - vk_op_argsort_push_constants pc2 = pc; - pc2.outer_start = 0; - pc2.outer_end = std::min(ncols_pad_log2, ctx->device->max_workgroup_size_log2); - pc2.inner_start = 0; - pc2.inner_end = 100; - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, subbuf1, dst_buf }, pc2, elements); - } - if (!use_small) { - ggml_vk_sync_buffers(ctx, subctx); - // Loop over outer/inner passes, synchronizing between each pass. - for (uint32_t outer = ctx->device->max_workgroup_size_log2; outer < ncols_pad_log2; ++outer) { - for (uint32_t inner = 0; inner < outer + 1; ++inner) { - vk_op_argsort_push_constants pc2 = pc; - pc2.outer_start = outer; - pc2.outer_end = outer + 1; - pc2.inner_start = inner; - pc2.inner_end = inner + 1; - // When the inner idx is large enough, there's only communication - // within a workgroup. So the remaining inner iterations can all - // run in the same dispatch. - if (outer - inner < pipeline_idx) { - pc2.inner_end = 100; - inner = outer; - pipeline = ctx->device->pipeline_argsort_large_f32[pipeline_idx]; - } else { - // Smaller workgroup empirically seems to perform better - pipeline = ctx->device->pipeline_argsort_large_f32[pipeline_idx - 2]; - } - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, subbuf1, dst_buf }, pc2, elements); - ggml_vk_sync_buffers(ctx, subctx); - } + uint32_t ne = (uint32_t)ggml_nelements(src0); + if (ggml_is_quantized(src0->type) && ggml_is_quantized(dst->type)) { + // Convert from number of logical elements to 2- or 4-byte units. + ne /= ggml_blck_size(src0->type); + if ((ggml_type_size(src0->type) % 4) == 0) { + ne *= ggml_type_size(src0->type) / 4; + } else { + ne *= ggml_type_size(src0->type) / 2; } - ctx->prealloc_x_need_sync = true; } -} - -static void ggml_vk_topk(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - uint32_t ncols = src0->ne[0]; - uint32_t nrows = ggml_nrows(src0); - uint32_t k = dst->ne[0]; - // tournament path is faster where it fits; use radix-select only past its k limit - const uint32_t k_min_pipeline = std::max((uint32_t) log2f(float(k)) + 1, ctx->device->subgroup_size_log2); - if (k_min_pipeline >= num_topk_pipelines || ctx->device->pipeline_topk_f32[k_min_pipeline] == nullptr) { - vk_pipeline pipeline = ctx->device->pipeline_topk_radix_f32; - GGML_ASSERT(pipeline != nullptr); + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ne); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CPY, std::move(p)); +} - if (ctx->prealloc_x_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } +void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); - vk_op_topk_radix_push_constants pc { ncols, k, nrows, 0, 0, 0 }; - std::array<uint32_t, 3> elements { - pipeline->wg_denoms[0], - std::min(nrows, ctx->device->properties.limits.maxComputeWorkGroupCount[1]), - 1, - }; - // the non-QSA path only uses bindings 0/1; bind valid buffers for the unused QSA slots - vk_subbuffer src0_buf = ggml_vk_tensor_subbuffer(ctx, src0); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - { src0_buf, dst_buf, src0_buf, src0_buf, src0_buf }, pc, elements); + // Skip empty skip_rows operations. For most ops the empty check at the start + // of ggml_vk_build_graph is sufficient, but set_rows can have a nonempty dst + // with empty srcs. + if (ggml_is_empty(src0) || ggml_is_empty(src1)) { return; } - vk_op_topk_push_constants pc { ncols, ncols, ncols, k, nrows, 0, 0 }; + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SET_ROWS, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, 0, + }); +} + +void ggml_vk_silu_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SILU_BACK, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f, 0.0f, 0.0f }); +} + +void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + float * op_params = (float *)dst->op_params; + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); + p.param1 = op_params[0]; + + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_NORM, std::move(p)); +} + +void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + const int * int_op_params = (const int *)dst->op_params; + const float * float_op_params = (const float *)dst->op_params; + + const uint32_t num_groups = int_op_params[0]; + const float eps = float_op_params[1]; + const uint32_t group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups); + + ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_GROUP_NORM, { group_size, 0, eps, 0.0f, 0.0f, 0.0f }); +} + +static uint32_t ggml_vk_rms_num_partials(ggml_backend_vk_context * ctx, const ggml_tensor *node) { + const uint32_t ne = (uint32_t)node->ne[0]; + const uint32_t denom = ctx->device->pipeline_add_rms[0][0][0]->wg_denoms[0]; + const uint32_t num_partials = CEIL_DIV(ne, denom); + return num_partials; +} + +uint32_t ggml_vk_rms_partials_size(ggml_backend_vk_context * ctx, const ggml_tensor *node) { + const uint32_t num_partials = ggml_vk_rms_num_partials(ctx, node); + const uint32_t num_bytes = ROUNDUP_POW2(num_partials * sizeof(uint32_t), ctx->device->partials_binding_alignment); + return num_bytes; +} - if (ctx->prealloc_x_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); +static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor *dst, const ggml_tensor *src0, const bool has_ff, bool backprop, const uint32_t set_rows_stride) { + const int n_dims = ((const int32_t *) dst->op_params)[1]; + const int mode = ((const int32_t *) dst->op_params)[2]; + const int n_offs = ((const int32_t *) dst->op_params)[15]; + // const int n_ctx = ((const int32_t *) dst->op_params)[3]; + const int n_ctx_orig = ((const int32_t *) dst->op_params)[4]; + const float freq_base = ((const float *) dst->op_params)[5]; + const float freq_scale = ((const float *) dst->op_params)[6]; + const float ext_factor = ((const float *) dst->op_params)[7]; + const float attn_factor = ((const float *) dst->op_params)[8]; + const float beta_fast = ((const float *) dst->op_params)[9]; + const float beta_slow = ((const float *) dst->op_params)[10]; + int sections[4] {}; + if (mode & GGML_ROPE_TYPE_MROPE) { + memcpy(sections, (const int32_t *) dst->op_params + 11, sizeof(int)*4); } - std::array<uint32_t, 3> elements; - elements[1] = std::min(nrows, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); - elements[2] = 1; + const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; - uint32_t num_elements = ncols; + float corr_dims[2]; + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); - // Each iteration reduces a workgroup's worth of elements down to the K - // largest elements. Repeat until we have the top K elements. - // Need to do at least one iteration to write out the results. - bool done_one_iter = false; - uint32_t dbl_buf_index = 0; - size_t dbl_buf_size; - while (num_elements > k || !done_one_iter) { + const float theta_scale = powf(freq_base, -2.0f/n_dims); - // Prefer going as small as num_topk_pipelines - 3 for perf reasons. - // But if K is larger, then we need a larger workgroup - uint32_t max_pipeline = num_topk_pipelines - 1; - uint32_t preferred_pipeline = std::max(num_topk_pipelines - 3, (uint32_t)log2f(float(k)) + 2); - max_pipeline = std::min(preferred_pipeline, max_pipeline); - uint32_t min_pipeline = (uint32_t)log2f(float(k)) + 1; - // require full subgroup - min_pipeline = std::max(min_pipeline, ctx->device->subgroup_size_log2); + uint32_t nb01 = src0->nb[1] / ggml_type_size(src0->type); + uint32_t nb02 = src0->nb[2] / ggml_type_size(src0->type); + uint32_t nb03 = src0->nb[3] / ggml_type_size(src0->type); - uint32_t pipeline_idx = (uint32_t)ceilf(log2f(float(num_elements))); - pipeline_idx = std::min(pipeline_idx, max_pipeline); - pipeline_idx = std::max(pipeline_idx, min_pipeline); + uint32_t nb11 = dst->nb[1] / ggml_type_size(dst->type); + uint32_t nb12 = dst->nb[2] / ggml_type_size(dst->type); + uint32_t nb13 = dst->nb[3] / ggml_type_size(dst->type); - if (num_elements > (1u << pipeline_idx)) { - // If we could finish on this loop iteration (i.e. a single workgroup) - // then do so. It's better than the overhead of another pass. - for (uint32_t i = pipeline_idx; i < num_topk_pipelines; ++i) { - if (num_elements <= (1u << i)) { - pipeline_idx = i; - break; - } - } - } + vk_op_rope_push_constants rope { + (uint32_t)mode, (uint32_t)ggml_nrows(src0), (uint32_t)n_dims, (uint32_t)n_offs, freq_scale, + freq_base, ext_factor, attn_factor, {corr_dims[0], corr_dims[1]}, theta_scale, has_ff, + { sections[0], sections[1], sections[2], sections[3] }, is_imrope, backprop, set_rows_stride, - vk_pipeline pipeline = ctx->device->pipeline_topk_f32[pipeline_idx]; - // If the device doesn't support a pipeline this large, use smaller - while (!pipeline) { - pipeline_idx--; - GGML_ASSERT(pipeline_idx >= min_pipeline); - pipeline = ctx->device->pipeline_topk_f32[pipeline_idx]; - } + (uint32_t)src0->ne[0], + (uint32_t)src0->ne[1], + (uint32_t)src0->ne[2], + nb01, nb02, nb03, + nb11, nb12, nb13, + 0, 0, // a_offset, d_offset filled in by init_pushconst_tensor_offsets + }; - vk_op_topk_push_constants pc2 = pc; - pc2.ncols_input = num_elements; + return rope; +} - // Number of elements remaining after this pass - uint32_t num_dst_elements = (num_elements / pipeline->wg_denoms[0]) * k + std::min(k, num_elements % pipeline->wg_denoms[0]); +static void ggml_vk_rms_norm_finish(ggml_backend_vk_context * ctx, const ggml_tensor * src0) { + if (ctx->do_add_rms_partials_offset_calculation) { + ctx->prealloc_size_add_rms_partials_offset += ggml_vk_rms_partials_size(ctx, src0); + ctx->do_add_rms_partials = false; + ctx->do_add_rms_partials_offset_calculation = false; + } +} - pc2.ncols_output = num_dst_elements; +void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, float * op_params) { + ggml_tensor * rms = cgraph->nodes[node_idx]; + const ggml_tensor * src0 = rms->src[0]; - if (!done_one_iter) { - // Reserve space for ivec2 per element, double buffered - // K per workgroup per row - dbl_buf_size = num_dst_elements * nrows * 2 * sizeof(int); - dbl_buf_size = ROUNDUP_POW2(dbl_buf_size, ctx->device->properties.limits.minStorageBufferOffsetAlignment); - const size_t x_sz = dbl_buf_size * 2; + if (ctx->fused_rms_norm_mode == RMS_NORM_VIEW_SET_ROWS) { + GGML_ASSERT(ctx->num_additional_fused_ops == 2); + ggml_tensor * set_rows = cgraph->nodes[node_idx + 2]; + const ggml_tensor * indices = set_rows->src[1]; + vk_op_binary_push_constants pc = ggml_vk_rms_norm_push_constants(src0, src0, set_rows, op_params[0], 0); + init_pushconst_tensor_offsets(ctx, pc, src0, src0, nullptr, nullptr, set_rows); - if (ctx->prealloc_size_x < x_sz) { - ctx->prealloc_size_x = x_sz; - ggml_vk_preallocate_buffers(ctx, subctx); - } - } + vk_pipeline pipeline = set_rows->type == GGML_TYPE_F16 ? + ctx->device->pipeline_rms_norm_set_rows_f32_f16 : ctx->device->pipeline_rms_norm_set_rows_f32_f32; + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + ggml_vk_tensor_subbuffer(ctx, src0, true), + ggml_vk_tensor_subbuffer(ctx, src0, true), + ggml_vk_tensor_subbuffer(ctx, set_rows, true), + ggml_vk_tensor_subbuffer(ctx, indices), + }, pc, { (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3] }); + ggml_vk_rms_norm_finish(ctx, src0); + return; + } - vk_subbuffer src_buf; - vk_subbuffer dst_buf; + if (ctx->fused_rms_norm_mode == RMS_NORM_MUL_ADD || ctx->fused_rms_norm_mode == RMS_NORM_MUL_ADD_MUL) { + ggml_tensor * mul = cgraph->nodes[node_idx + 1]; + ggml_tensor * add = cgraph->nodes[node_idx + 2]; + const ggml_tensor * weight = mul->src[0] == rms ? mul->src[1] : mul->src[0]; + const ggml_tensor * residual = add->src[0] == mul ? add->src[1] : add->src[0]; + const bool do_post_multiply = ctx->fused_rms_norm_mode == RMS_NORM_MUL_ADD_MUL; + GGML_ASSERT(ctx->num_additional_fused_ops == (do_post_multiply ? 3 : 2)); + ggml_tensor * dst = do_post_multiply ? cgraph->nodes[node_idx + 3] : add; + const ggml_tensor * post_scale = do_post_multiply ? + (dst->src[0] == add ? dst->src[1] : dst->src[0]) : src0; - if (num_elements == ncols) { - pc2.first_pass = 1; - src_buf = ggml_vk_tensor_subbuffer(ctx, src0); - } else { - src_buf = { ctx->prealloc_x, dbl_buf_index * dbl_buf_size, dbl_buf_size }; - } - if (num_dst_elements == k) { - pc2.last_pass = 1; - dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); + const uint32_t num_partials = ctx->do_add_rms_partials ? ggml_vk_rms_num_partials(ctx, dst) : 0; + vk_op_binary_push_constants pc = ggml_vk_rms_norm_push_constants(src0, weight, dst, op_params[0], num_partials); + init_pushconst_tensor_offsets(ctx, pc, src0, weight, residual, post_scale, dst); + + vk_pipeline pipeline; + if (ctx->do_add_rms_partials) { + pipeline = do_post_multiply ? + ctx->device->pipeline_rms_norm_mul_add_mul_partials_f32 : ctx->device->pipeline_rms_norm_mul_add_partials_f32; } else { - dst_buf = { ctx->prealloc_x, (dbl_buf_index ^ 1) * dbl_buf_size, dbl_buf_size }; + pipeline = do_post_multiply ? + ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32; } - - elements[0] = num_elements; - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src_buf, dst_buf }, pc2, elements); - num_elements = num_dst_elements; - dbl_buf_index ^= 1; - if (num_elements > k) { - ggml_vk_sync_buffers(ctx, subctx); - } - done_one_iter = true; + if (ctx->do_add_rms_partials) { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + ggml_vk_tensor_subbuffer(ctx, src0, true), + ggml_vk_tensor_subbuffer(ctx, weight, true), + ggml_vk_tensor_subbuffer(ctx, dst, true), + ggml_vk_subbuffer(ctx, ctx->prealloc_add_rms_partials, ctx->prealloc_size_add_rms_partials_offset), + ggml_vk_tensor_subbuffer(ctx, residual), + ggml_vk_tensor_subbuffer(ctx, post_scale), + }, pc, { (uint32_t)CEIL_DIV(src0->ne[0], 128), 1, 1 }); + } else { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + ggml_vk_tensor_subbuffer(ctx, src0, true), + ggml_vk_tensor_subbuffer(ctx, weight, true), + ggml_vk_tensor_subbuffer(ctx, dst, true), + ggml_vk_tensor_subbuffer(ctx, residual), + ggml_vk_tensor_subbuffer(ctx, post_scale), + }, pc, { (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3] }); + } + ggml_vk_rms_norm_finish(ctx, src0); + return; } - ctx->prealloc_x_need_sync = true; -} - -static void ggml_vk_topk_qsa(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx) { - const ggml_tensor * get_rows = cgraph->nodes[node_idx + 0]; - const ggml_tensor * add = cgraph->nodes[node_idx + ctx->num_additional_fused_ops - 1]; - ggml_tensor * top_k = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; - const ggml_tensor * scores = get_rows->src[0]; // [n_tps, n_blocks, n_stream] - const ggml_tensor * cell_blk = get_rows->src[1]; // [n_kv, n_stream] + ggml_tensor * dst; + const ggml_tensor * src1; - // raw f16 mask: follow the reshape/cpy chain back to the materialized input - const ggml_tensor * mask = add->src[1]; - while (mask->op == GGML_OP_RESHAPE || mask->op == GGML_OP_CPY) { - mask = mask->src[0]; + if (ctx->fused_rms_norm_mode != RMS_NORM_COUNT) { + ggml_tensor * mul = cgraph->nodes[node_idx + 1]; + dst = mul; + src1 = mul->src[0] == rms ? mul->src[1] : mul->src[0]; + } else { + dst = rms; + src1 = src0; } - const uint32_t n_tps = scores->ne[0]; - const uint32_t n_blocks = scores->ne[1]; - const uint32_t n_stream = scores->ne[2]; - const uint32_t n_kv = cell_blk->ne[0]; - const uint32_t width = top_k->ne[0]; - const uint32_t nrows = n_tps * n_stream; - - vk_pipeline pipeline = ctx->device->pipeline_topk_radix_qsa; - GGML_ASSERT(pipeline != nullptr); + const uint32_t num_partials = ctx->do_add_rms_partials ? ggml_vk_rms_num_partials(ctx, dst) : 0; + vk_op_binary_push_constants bin = ggml_vk_rms_norm_push_constants(src0, src1, dst, op_params[0], num_partials); - // scratch holds the gathered+masked input, materialized once and reused across passes - const size_t scratch_size = size_t{ n_kv } * nrows * sizeof(float); - if (ctx->prealloc_size_x < scratch_size) { - ctx->prealloc_size_x = scratch_size; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if (ctx->prealloc_x_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } + if (ctx->fused_rms_norm_mode == RMS_NORM_MUL_ROPE || + ctx->fused_rms_norm_mode == RMS_NORM_MUL_ROPE_VIEW_SET_ROWS) { + static constexpr uint32_t max_tensors = 7; + const ggml_tensor *tensors[max_tensors] {}; - vk_op_topk_radix_push_constants pc { n_kv, width, nrows, n_tps, n_blocks, n_stream }; - std::array<uint32_t, 3> elements { - pipeline->wg_denoms[0], - std::min(nrows, ctx->device->properties.limits.maxComputeWorkGroupCount[1]), - 1, - }; - vk_subbuffer scratch_buf { ctx->prealloc_x, 0, ctx->prealloc_x->size }; - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - { ggml_vk_tensor_subbuffer(ctx, scores), ggml_vk_tensor_subbuffer(ctx, top_k), - ggml_vk_tensor_subbuffer(ctx, cell_blk), ggml_vk_tensor_subbuffer(ctx, mask), - scratch_buf }, pc, elements); - ctx->prealloc_x_need_sync = true; -} + ggml_tensor *rms = cgraph->nodes[node_idx + 0]; + ggml_tensor *mul = cgraph->nodes[node_idx + 1]; + ggml_tensor *rope = cgraph->nodes[node_idx + 2]; -static void ggml_vk_sum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, ggml_nelements(src0)); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SUM, p); -} + ggml_tensor *other_src = mul->src[0] == rms ? mul->src[1] : mul->src[0]; -static void ggml_vk_sum_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SUM_ROWS, p); -} + bool do_set_rows = ctx->fused_rms_norm_mode == RMS_NORM_MUL_ROPE_VIEW_SET_ROWS; + GGML_ASSERT(ctx->num_additional_fused_ops == (do_set_rows ? 4 : 2)); -static void ggml_vk_mean(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]); - p.weight = 1.0f / (float)src0->ne[0]; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_MEAN, p); -} + tensors[0] = rms->src[0]; + tensors[1] = other_src; + tensors[2] = mul; + tensors[3] = rope->src[1]; // pos + tensors[4] = rope->src[2]; // ff + tensors[5] = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; // dst + tensors[6] = do_set_rows ? tensors[5]->src[1] : nullptr; + const uint32_t set_rows_stride = do_set_rows ? tensors[5]->nb[1] / ggml_type_size(tensors[5]->type) : 0; -static void ggml_vk_cumsum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - vk_op_sum_rows_push_constants pc = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]); - // Use the single pass shader when the rows are small or there are enough rows to fill the GPU. - // For fewer, larger rows, use the multipass shader to spread each row across SMs. - if (dst->ne[0] <= 4096 || ggml_nrows(dst) >= ctx->device->shader_core_count) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CUMSUM, pc); - return; - } + vk_op_rms_norm_mul_rope_push_constants pc; + pc.bin = bin; + pc.rope = ggml_vk_make_rope_constants(rope, rope->src[0], tensors[4] != nullptr, false, set_rows_stride); - // First pass computes partial sums within a block, and stores the last partial - // to the temp buffer. Second pass sums the block partials from the temp buffer - // and adds that to the result of the first pass. - vk_pipeline pipeline1 = ctx->device->pipeline_cumsum_multipass1_f32; - vk_pipeline pipeline2 = ctx->device->pipeline_cumsum_multipass2_f32; - GGML_ASSERT(pipeline1 != nullptr && pipeline2 != nullptr); + vk_pipeline pipeline = tensors[5]->type == GGML_TYPE_F16 ? ctx->device->pipeline_rms_norm_mul_rope_f32_f16 : ctx->device->pipeline_rms_norm_mul_rope_f32_f32; - ggml_pipeline_request_descriptor_sets(ctx, pipeline1, 1); - ggml_pipeline_request_descriptor_sets(ctx, pipeline2, 1); + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - std::array<uint32_t, 3> elements; + ggml_backend_vk_buffer_context * buf_ctx[max_tensors]; + vk_buffer buf[max_tensors]; + size_t offset[max_tensors]; + bool uma[max_tensors]; - elements[0] = dst->ne[0]; - elements[1] = (uint32_t)ggml_nrows(dst); - elements[2] = 1; + for (uint32_t i = 0; i < max_tensors; ++i) { + if (!tensors[i]) { + // If any remaining descriptors are unused, just point them at src[0] + buf[i] = buf[0]; + offset[i] = 0; + continue; + } + buf_ctx[i] = (ggml_backend_vk_buffer_context *)tensors[i]->buffer->context; + buf[i] = nullptr; + offset[i] = 0; + uma[i] = false; - size_t temp_size = sizeof(float) * elements[0] * ggml_nrows(dst); + if (ctx->device->uma) { + ggml_vk_host_get(ctx->device, tensors[i]->data, buf[i], offset[i]); + uma[i] = buf[i] != nullptr; + } + if (!uma[i]) { + buf[i] = buf_ctx[i]->dev_buffer; + offset[i] = vk_tensor_offset(tensors[i]) + tensors[i]->view_offs; + } + GGML_ASSERT(buf[i] != nullptr); + } - if (ctx->prealloc_size_split_k < temp_size) { - ctx->prealloc_size_split_k = temp_size; - ggml_vk_preallocate_buffers(ctx, subctx); - } + // a_offset is unused (the fused path reads from shared memory), but the rope/set_rows dst can be misaligned. + // Round the binding offset down to the storage buffer alignment; the in-element shift goes in pc.rope.d_offset. + pc.rope.d_offset = get_misalign_bytes(ctx, tensors[5]) / ggml_type_size(tensors[5]->type); + offset[5] &= ~(size_t(ctx->device->properties.limits.minStorageBufferOffsetAlignment) - 1); - vk_subbuffer src_buf = ggml_vk_tensor_subbuffer(ctx, src0); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); - vk_subbuffer temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_split_k, 0); + std::array<uint32_t, 3> elements; + elements = { (uint32_t)rms->src[0]->ne[1], (uint32_t)rms->src[0]->ne[2], (uint32_t)rms->src[0]->ne[3] }; - if (ctx->prealloc_split_k_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); + static_assert(max_tensors == 7); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + ggml_vk_subbuffer(ctx, buf[0], offset[0]), + ggml_vk_subbuffer(ctx, buf[1], offset[1]), + ggml_vk_subbuffer(ctx, buf[2], offset[2]), + ggml_vk_subbuffer(ctx, buf[3], offset[3]), + ggml_vk_subbuffer(ctx, buf[4], offset[4]), + ggml_vk_subbuffer(ctx, buf[5], offset[5]), + ggml_vk_subbuffer(ctx, buf[6], offset[6]), + }, pc, elements); + } else { + GGML_ASSERT(ctx->fused_rms_norm_mode == RMS_NORM_MUL || ctx->fused_rms_norm_mode == RMS_NORM_COUNT); + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_RMS_NORM, std::move(bin)); } - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline1, {src_buf, dst_buf, temp_buf}, pc, elements); - ggml_vk_sync_buffers(ctx, subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline2, {src_buf, dst_buf, temp_buf}, pc, elements); + ggml_vk_rms_norm_finish(ctx, src0); +} - ctx->prealloc_split_k_need_sync = true; +void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + float * op_params = (float *)dst->op_params; + ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_RMS_NORM_BACK, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f, 0.0f, 0.0f }); } -static std::array<uint32_t, 3> ggml_vk_nrows_elements(uint32_t nr) { - if (nr > 262144) { - return { 512, 512, CEIL_DIV(nr, 262144) }; - } - if (nr > 512) { - return { 512, CEIL_DIV(nr, 512), 1 }; - } - return { nr, 1, 1 }; +void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + const float * op_params = (const float *)dst->op_params; + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); + p.param1 = op_params[0]; + ggml_vk_op_f32<vk_op_unary_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_L2_NORM, std::move(p)); } -static void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; +static void ggml_vk_sinkhorn_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + float eps; + memcpy(&eps, dst->op_params, sizeof(eps)); + const int32_t iters = ggml_get_op_params_i32(dst, 1); + const int64_t n = src0->ne[0]; - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(eps >= 0.0f); + GGML_ASSERT(iters >= 1); GGML_ASSERT(ggml_is_contiguous(src0)); - GGML_ASSERT(ggml_is_contiguous(src1)); GGML_ASSERT(ggml_is_contiguous(dst)); - GGML_ASSERT(ggml_are_same_shape(src0, src1)); - GGML_ASSERT(ggml_is_scalar(dst)); + GGML_ASSERT(ggml_are_same_shape(src0, dst)); + GGML_ASSERT(n == src0->ne[1]); + GGML_ASSERT(n == 2 || n == 4 || n == 8); - const uint32_t nclasses = (uint32_t)src0->ne[0]; - const uint32_t nrows = (uint32_t)ggml_nrows(src0); + ggml_vk_op_f32<vk_op_sinkhorn_norm_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SINKHORN_NORM, { + (uint32_t) n, + (uint32_t) (src0->ne[2] * src0->ne[3]), + eps, + iters, + 0, + 0, + }); +} - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, nullptr, dst, GGML_OP_CROSS_ENTROPY_LOSS); - GGML_ASSERT(pipeline != nullptr); +void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, vk_op_unary_push_constants_init(src0, dst)); +} - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_sum_rows_f32, 1); +void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + float * op_params = (float *)dst->op_params; + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); + p.param1 = op_params[1]; + p.param2 = op_params[2]; + p.param3 = op_params[3]; + p.param4 = op_params[4]; + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, std::move(p)); +} - vk_subbuffer src0_buf = ggml_vk_tensor_subbuffer(ctx, src0); - vk_subbuffer src1_buf = ggml_vk_tensor_subbuffer(ctx, src1); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); +void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const float * op_params_f = (const float *)dst->op_params; - const vk_op_push_constants pc = { nclasses, nrows, 0.0f, 0.0f, 0.0f, 0.0f }; + const bool swapped = (bool)dst->op_params[1]; + const bool split = src1 != nullptr; + const float alpha = op_params_f[2]; + const float limit = op_params_f[3]; - const size_t tmp_size = (size_t)nrows * sizeof(float); - if (ctx->prealloc_size_x < tmp_size) { - ctx->prealloc_size_x = tmp_size; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if (ctx->prealloc_x_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); + if (!split) { + GGML_ASSERT(src0->ne[0] / 2 == dst->ne[0]); + } else { + GGML_ASSERT(src0->ne[0] == src1->ne[0]); + GGML_ASSERT(src0->ne[0] == dst->ne[0]); + GGML_ASSERT(src0->type == src1->type); } - vk_subbuffer tmp_buf = { ctx->prealloc_x, 0, tmp_size }; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, tmp_buf }, pc, ggml_vk_nrows_elements(nrows)); - ggml_vk_sync_buffers(ctx, subctx); - - vk_op_sum_rows_push_constants sp = {}; - sp.n_cols = nrows; - sp.ne01 = 1; - sp.ne02 = 1; - sp.weight = 1.0f; - init_pushconst_fastdiv(sp); - sp.misalign_offsets = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); + const uint32_t mode = split ? 2 : (swapped ? 1 : 0); + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = split ? ggml_type_size(src1->type) : src0_type_size; + const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_sum_rows_f32, { tmp_buf, dst_buf }, sp, { 1, 1, 1 }); - ctx->prealloc_x_need_sync = true; + ggml_vk_op_f32<vk_op_glu_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_GLU, + { + (uint32_t)ggml_nelements(dst), + (uint32_t)src0->ne[0], + (uint32_t)dst->ne[0], + mode, + alpha, + limit, + (uint32_t)(src0->nb[0] / src0_type_size), + (uint32_t)(src0->nb[1] / src0_type_size), + (uint32_t)(src0->nb[2] / src0_type_size), + (uint32_t)(src0->nb[3] / src0_type_size), + (uint32_t)((split ? src1->nb[0] : src0->nb[0]) / src1_type_size), + (uint32_t)((split ? src1->nb[1] : src0->nb[1]) / src1_type_size), + (uint32_t)((split ? src1->nb[2] : src0->nb[2]) / src1_type_size), + (uint32_t)((split ? src1->nb[3] : src0->nb[3]) / src1_type_size), + (uint32_t)(dst->nb[0] / dst_type_size), + (uint32_t)(dst->nb[1] / dst_type_size), + (uint32_t)(dst->nb[2] / dst_type_size), + (uint32_t)(dst->nb[3] / dst_type_size), + (uint32_t)dst->ne[1], + (uint32_t)dst->ne[2], + 0, + 0, 0, 0, 0, 0, 0, + }); } -static void ggml_vk_cross_entropy_loss_back(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { - const ggml_tensor * grad = dst->src[0]; - const ggml_tensor * logits = dst->src[1]; - const ggml_tensor * labels = dst->src[2]; - - GGML_ASSERT(grad->type == GGML_TYPE_F32); - GGML_ASSERT(logits->type == GGML_TYPE_F32); - GGML_ASSERT(labels->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_scalar(grad)); - GGML_ASSERT(ggml_is_contiguous(grad)); - GGML_ASSERT(ggml_is_contiguous(logits)); - GGML_ASSERT(ggml_is_contiguous(labels)); - GGML_ASSERT(ggml_is_contiguous(dst)); - GGML_ASSERT(ggml_are_same_shape(logits, labels)); - GGML_ASSERT(ggml_are_same_shape(logits, dst)); - - const uint32_t nclasses = (uint32_t)logits->ne[0]; - const uint32_t nrows = (uint32_t)ggml_nrows(logits); - - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, grad, logits, labels, dst, GGML_OP_CROSS_ENTROPY_LOSS_BACK); - GGML_ASSERT(pipeline != nullptr); - - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); +void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + int32_t * op_params = (int32_t *)dst->op_params; + ggml_vk_op_f32<vk_op_diag_mask_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); +} - vk_subbuffer grad_buf = ggml_vk_tensor_subbuffer(ctx, grad); - vk_subbuffer logits_buf = ggml_vk_tensor_subbuffer(ctx, logits); - vk_subbuffer labels_buf = ggml_vk_tensor_subbuffer(ctx, labels); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); +void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { + float * op_params = (float *)dst->op_params; - const vk_op_push_constants pc = { nclasses, nrows, 0.0f, 0.0f, 0.0f, 0.0f }; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { grad_buf, logits_buf, labels_buf, dst_buf }, pc, ggml_vk_nrows_elements(nrows)); -} + float scale = op_params[0]; + float max_bias = op_params[1]; -static void ggml_vk_argmax(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_ARGMAX, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], 0.0f, 0.0f, 0.0f, 0.0f }); -} + const uint32_t ncols = (uint32_t)src0->ne[0]; + const uint32_t nrows_x = (uint32_t)ggml_nrows(src0); + const uint32_t nrows_y = (uint32_t)src0->ne[1]; -static void ggml_vk_count_equal(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_COUNT_EQUAL, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f, 0.0f, 0.0f }); -} + const uint32_t ne12 = src1 ? (uint32_t)(src1->ne[2]) : 0u; + const uint32_t ne13 = src1 ? (uint32_t)(src1->ne[3]) : 0u; + const uint32_t nb11 = src1 ? (uint32_t)(src1->nb[1] / src1->nb[0]) : 0u; + const uint32_t nb12 = src1 ? (uint32_t)(src1->nb[2] / src1->nb[0]) : 0u; + const uint32_t nb13 = src1 ? (uint32_t)(src1->nb[3] / src1->nb[0]) : 0u; -static void ggml_vk_solve_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); + const uint32_t n_head_kv = src0->ne[2]; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv)); - ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SOLVE_TRI, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - 0.0f, 0.0f, 0, - }); -} + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); -static void ggml_vk_im2col(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const int32_t s0 = dst->op_params[0]; - const int32_t s1 = dst->op_params[1]; - const int32_t p0 = dst->op_params[2]; - const int32_t p1 = dst->op_params[3]; - const int32_t d0 = dst->op_params[4]; - const int32_t d1 = dst->op_params[5]; + vk_op_soft_max_push_constants pc { + ncols, + src1 != nullptr ? nrows_y : (uint32_t)0, + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], + ne12, ne13, + nb11, nb12, nb13, + scale, max_bias, + m0, m1, + n_head_log2, + nrows_x, + src2 != nullptr + }; - const bool is_2D = dst->op_params[6] == 1; + if (ncols <= 16384) { + ggml_vk_op_f32<vk_op_soft_max_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_SOFT_MAX, std::move(pc)); + } else { - const uint32_t IC = src1->ne[is_2D ? 2 : 1]; - const uint32_t IH = is_2D ? src1->ne[1] : 1; - const uint32_t IW = src1->ne[0]; + vk_subbuffer buf_a = ggml_vk_tensor_subbuffer(ctx, src0); + vk_subbuffer buf_b = src1 ? ggml_vk_tensor_subbuffer(ctx, src1) : buf_a; + vk_subbuffer buf_c = src2 ? ggml_vk_tensor_subbuffer(ctx, src2) : buf_a; + vk_subbuffer buf_d = ggml_vk_tensor_subbuffer(ctx, dst); - const uint32_t KH = is_2D ? src0->ne[1] : 1; - const uint32_t KW = src0->ne[0]; + uint32_t elems_per_wg = 128 * 4; + uint32_t num_wgs = CEIL_DIV(ncols, elems_per_wg); + size_t tmp_size = num_wgs * nrows_x * sizeof(float); - const uint32_t OH = is_2D ? dst->ne[2] : 1; - const uint32_t OW = dst->ne[1]; + if (ctx->prealloc_size_x < tmp_size) { + ctx->prealloc_size_x = tmp_size; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if (ctx->prealloc_size_y < tmp_size) { + ctx->prealloc_size_y = tmp_size; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if (ctx->prealloc_x_need_sync || ctx->prealloc_y_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } - const uint32_t offset_delta = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32 - const uint32_t batch_offset = src1->nb[is_2D ? 3 : 2] / 4; // nb is byte offset, src is type float32 + vk_subbuffer buf_x = { ctx->prealloc_x, 0, tmp_size }; + vk_subbuffer buf_y = { ctx->prealloc_y, 0, tmp_size }; - const uint32_t batch = src1->ne[is_2D ? 3 : 2]; + std::array<uint32_t, 3> elements = { num_wgs, nrows_x, 1 }; - const ggml_backend_vk_buffer_context * d_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; - const vk_buffer d_buf = d_buf_ctx->dev_buffer; + vk_pipeline pipeline1 = src1 && src1->type == GGML_TYPE_F16 ? ctx->device->pipeline_soft_max_large1_f32_f16 : ctx->device->pipeline_soft_max_large1_f32; + vk_pipeline pipeline2 = src1 && src1->type == GGML_TYPE_F16 ? ctx->device->pipeline_soft_max_large2_f32_f16 : ctx->device->pipeline_soft_max_large2_f32; + vk_pipeline pipeline3 = src1 && src1->type == GGML_TYPE_F16 ? ctx->device->pipeline_soft_max_large3_f32_f16 : ctx->device->pipeline_soft_max_large3_f32; - const vk::DeviceAddress dst_addr = d_buf->bda_addr + vk_tensor_offset(dst) + dst->view_offs; + ggml_pipeline_request_descriptor_sets(ctx, pipeline1, 1); + ggml_pipeline_request_descriptor_sets(ctx, pipeline2, 1); + ggml_pipeline_request_descriptor_sets(ctx, pipeline3, 1); - ggml_vk_op_f32<vk_op_im2col_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_IM2COL, { - dst_addr, - batch_offset, offset_delta, - IC, IW, IH, OW, OH, KW, KH, - OH * batch, - IC * KH * KW, - s0, s1, p0, p1, d0, d1, batch * IC - }); -} + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline1, { buf_a, buf_b, buf_c, buf_d, buf_x, buf_y }, pc, elements); + ggml_vk_sync_buffers(ctx, subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline2, { buf_a, buf_b, buf_c, buf_d, buf_x, buf_y }, pc, elements); + ggml_vk_sync_buffers(ctx, subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline3, { buf_a, buf_b, buf_c, buf_d, buf_x, buf_y }, pc, elements); -static void ggml_vk_im2col_3d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_TENSOR_BINARY_OP_LOCALS + ctx->prealloc_x_need_sync = true; + ctx->prealloc_y_need_sync = true; + } +} - const int32_t s0 = ((const int32_t *)(dst->op_params))[0]; - const int32_t s1 = ((const int32_t *)(dst->op_params))[1]; - const int32_t s2 = ((const int32_t *)(dst->op_params))[2]; - const int32_t p0 = ((const int32_t *)(dst->op_params))[3]; - const int32_t p1 = ((const int32_t *)(dst->op_params))[4]; - const int32_t p2 = ((const int32_t *)(dst->op_params))[5]; - const int32_t d0 = ((const int32_t *)(dst->op_params))[6]; - const int32_t d1 = ((const int32_t *)(dst->op_params))[7]; - const int32_t d2 = ((const int32_t *)(dst->op_params))[8]; - const int32_t IC = ((const int32_t *)(dst->op_params))[9]; +void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + float * op_params = (float *)dst->op_params; + ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SOFT_MAX_BACK, { (uint32_t)src0->ne[0], (uint32_t)ggml_nrows(src0), op_params[0], op_params[1], 0.0f, 0.0f }); +} - const int64_t N = ne13 / IC; - const int64_t ID = ne12; - const int64_t IH = ne11; - const int64_t IW = ne10; +void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) { + topk_moe_mode mode = ctx->fused_topk_moe_mode; + const bool has_bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS || mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS; + ggml_tensor * logits = cgraph->nodes[node_idx + 0]->src[0]; + ggml_tensor * bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS ? cgraph->nodes[node_idx + 2]->src[1] : + mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? cgraph->nodes[node_idx + 3]->src[1] : + logits; + ggml_tensor * weights = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; + ggml_tensor * ids = mode == TOPK_MOE_SIGMOID_NORM_BIAS ? cgraph->nodes[node_idx + 4] : + mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? cgraph->nodes[node_idx + 5] : + mode == TOPK_MOE_LATE_SOFTMAX ? cgraph->nodes[node_idx + 1] : + cgraph->nodes[node_idx + 3]; - const int64_t KD = ne02; - const int64_t KH = ne01; - const int64_t KW = ne00; + GGML_ASSERT(logits->type == GGML_TYPE_F32); + GGML_ASSERT(bias->type == GGML_TYPE_F32); + GGML_ASSERT(weights->type == GGML_TYPE_F32); + GGML_ASSERT(ids->type == GGML_TYPE_I32); - const int64_t OD = ne3 / N; - const int64_t OH = ne2; - const int64_t OW = ne1; + const int n_experts = logits->ne[0]; + const int n_rows = logits->ne[1]; + const int n_expert_used = weights->ne[1]; - const ggml_backend_vk_buffer_context * d_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; - const vk_buffer d_buf = d_buf_ctx->dev_buffer; + GGML_ASSERT(ids->nb[1] / ggml_type_size(ids->type) == (size_t) n_experts); - const vk::DeviceAddress dst_addr = d_buf->bda_addr + vk_tensor_offset(dst) + dst->view_offs; + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, cgraph->nodes[node_idx], GGML_OP_SOFT_MAX); - vk_op_im2col_3d_push_constants pc {}; + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - pc.dst_addr = dst_addr; - pc.nb10 = nb10 / ggml_type_size(src1->type); - pc.nb11 = nb11 / ggml_type_size(src1->type); - pc.nb12 = nb12 / ggml_type_size(src1->type); - pc.nb13 = nb13 / ggml_type_size(src1->type); - pc.s0 = s0; - pc.s1 = s1; - pc.s2 = s2; - pc.p0 = p0; - pc.p1 = p1; - pc.p2 = p2; - pc.d0 = d0; - pc.d1 = d1; - pc.d2 = d2; - pc.IW = IW; - pc.IH = IH; - pc.ID = ID; - pc.IC = IC; - pc.KW = KW; - pc.OH = OH; - pc.KD_KH_KW = KD*KH*KW; - pc.KH_KW = KH*KW; - pc.IC_KD_KH_KW = IC*KD*KH*KW; - pc.N_OD_OH = N*OD*OH; - pc.OD_OH = OD*OH; - pc.OD_OH_OW_IC_KD_KH_KW = OD*OH*OW*IC*KD*KH*KW; - pc.OH_OW_IC_KD_KH_KW = OH*OW*IC*KD*KH*KW; - pc.OW_IC_KD_KH_KW = OW*IC*KD*KH*KW; + vk_subbuffer logits_buf = ggml_vk_tensor_subbuffer(ctx, logits); + vk_subbuffer bias_buf = ggml_vk_tensor_subbuffer(ctx, bias); + vk_subbuffer weights_buf = ggml_vk_tensor_subbuffer(ctx, weights); + vk_subbuffer ids_buf = ggml_vk_tensor_subbuffer(ctx, ids); - ggml_vk_op_f32<vk_op_im2col_3d_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_IM2COL_3D, std::move(pc)); -} + vk_op_topk_moe_push_constants pc {}; + pc.n_rows = n_rows; + pc.n_experts_push = n_experts; + pc.n_expert_used = n_expert_used; + pc.clamp_min = -std::numeric_limits<float>::infinity(); + pc.clamp_max = std::numeric_limits<float>::infinity(); + if (mode == TOPK_MOE_EARLY_SOFTMAX_NORM) { + ggml_tensor * clamp = cgraph->nodes[node_idx + 7]; + GGML_ASSERT(clamp->op == GGML_OP_CLAMP); + pc.clamp_min = ggml_get_op_params_f32(clamp, 0); + pc.clamp_max = ggml_get_op_params_f32(clamp, 1); + } + if (mode == TOPK_MOE_SIGMOID_NORM_BIAS) { + ggml_tensor * clamp = cgraph->nodes[node_idx + 8]; + GGML_ASSERT(clamp->op == GGML_OP_CLAMP); + pc.clamp_min = ggml_get_op_params_f32(clamp, 0); + pc.clamp_max = ggml_get_op_params_f32(clamp, 1); + } + if (mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS) { + ggml_tensor * clamp = cgraph->nodes[node_idx + 9]; + GGML_ASSERT(clamp->op == GGML_OP_CLAMP); + pc.clamp_min = ggml_get_op_params_f32(clamp, 0); + pc.clamp_max = ggml_get_op_params_f32(clamp, 1); + } -static void ggml_vk_timestep_embedding(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - const uint32_t dim = dst->op_params[0]; - const uint32_t max_period = dst->op_params[1]; - const uint32_t nb1 = dst->nb[1] / ggml_type_size(dst->type); +#define GATING_FUNC_SOFTMAX 0 +#define GATING_FUNC_SIGMOID 1 +#define GATING_FUNC_SOFTMAX_WEIGHT 2 +#define GATING_FUNC_SQRT_SOFTPLUS 3 - ggml_vk_op_f32<vk_op_timestep_embedding_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_TIMESTEP_EMBEDDING, { - nb1, dim, max_period, - }); -} + pc.gating_func = mode == TOPK_MOE_SIGMOID_NORM_BIAS ? GATING_FUNC_SIGMOID : + mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? GATING_FUNC_SQRT_SOFTPLUS : + mode == TOPK_MOE_LATE_SOFTMAX ? GATING_FUNC_SOFTMAX_WEIGHT : + GATING_FUNC_SOFTMAX; + pc.has_bias = has_bias; + pc.with_norm = mode == TOPK_MOE_EARLY_SOFTMAX_NORM || has_bias; + if (ctx->fused_topk_moe_scale) { + GGML_ASSERT(weights->op == GGML_OP_SCALE); + pc.output_scale = ggml_get_op_params_f32(weights, 0); + pc.output_bias = ggml_get_op_params_f32(weights, 1); + } else { + pc.output_scale = 1.0f; + pc.output_bias = 0.0f; + } -static void ggml_vk_conv_transpose_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - // src0: (K, Cout, Cin, 1) -- kernel - // src1: (L, Cin, 1, 1) -- input - // dst: (*, Cout, 1, 1) + GGML_ASSERT(n_expert_used <= n_experts); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); + const uint32_t rows_per_block = 4; + std::array<uint32_t, 3> elements = { CEIL_DIV(n_rows, rows_per_block), 1, 1 }; - GGML_TENSOR_BINARY_OP_LOCALS + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, {logits_buf, bias_buf, weights_buf, ids_buf}, pc, elements); +} - GGML_ASSERT(nb00 == sizeof(float)); - GGML_ASSERT(nb10 == sizeof(float)); +void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx, bool backprop) { + ggml_tensor * dst = cgraph->nodes[node_idx]; + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * src2 = dst->src[2]; + const ggml_tensor * src3 = nullptr; + const int n_dims = ((int32_t *) dst->op_params)[1]; + const int mode = ((int32_t *) dst->op_params)[2]; + // const int n_ctx = ((int32_t *) dst->op_params)[3]; + const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; + const float freq_base = ((float *) dst->op_params)[5]; + const float beta_fast = ((float *) dst->op_params)[9]; + const float beta_slow = ((float *) dst->op_params)[10]; + int sections[4] {}; + if (mode & GGML_ROPE_TYPE_MROPE) { + memcpy(sections, (int32_t *) dst->op_params + 11, sizeof(int)*4); + } - const int32_t s0 = dst->op_params[0]; + float corr_dims[2]; + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); - vk_op_conv_transpose_1d_push_constants p{}; - p.Cout = static_cast<uint32_t>(ne01); - p.Cin = static_cast<uint32_t>(ne02); - p.K = static_cast<uint32_t>(ne00); - p.L = static_cast<uint32_t>(ne10); - p.KL = static_cast<uint32_t>(ne0); - p.nb01 = static_cast<uint32_t>(nb01 / nb00); - p.nb02 = static_cast<uint32_t>(nb02 / nb00); - p.nb11 = static_cast<uint32_t>(nb11 / nb10); - p.nb1 = static_cast<uint32_t>(nb1 / nb0); - p.s0 = static_cast<uint32_t>(s0); + uint32_t set_rows_stride = 0; + // Fused rope + view + set_rows passes the set_rows destination stride in set_rows_stride + // and overrides the dst and sets src3=row_indices + if (ctx->num_additional_fused_ops > 0) { + set_rows_stride = cgraph->nodes[node_idx + 2]->nb[1] / ggml_type_size(cgraph->nodes[node_idx + 2]->type); + src3 = cgraph->nodes[node_idx + 2]->src[1]; + dst = cgraph->nodes[node_idx + 2]; + } - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_TRANSPOSE_1D, std::move(p)); + ggml_vk_op_f32<vk_op_rope_push_constants>(ctx, subctx, src0, src1, src2, src3, dst, GGML_OP_ROPE, + ggml_vk_make_rope_constants(cgraph->nodes[node_idx], src0, src2 != nullptr, backprop, set_rows_stride)); } -static void ggml_vk_col2im_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - // src0: [K_OC, T_in] columns from matmul - // dst: [T_out, OC] - - const int32_t stride = dst->op_params[0]; - const int32_t oc = dst->op_params[1]; - const int32_t p0 = dst->op_params[2]; - - const uint32_t K_OC = static_cast<uint32_t>(src0->ne[0]); - const uint32_t T_in = static_cast<uint32_t>(src0->ne[1]); - const uint32_t T_out = static_cast<uint32_t>(dst->ne[0]); - const uint32_t OC = static_cast<uint32_t>(oc); - const uint32_t K = K_OC / OC; +void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + const uint32_t * op_params = (const uint32_t *)dst->op_params; - vk_op_col2im_1d_push_constants p{}; - p.T_out = T_out; - p.OC = OC; - p.K_OC = K_OC; - p.T_in = T_in; - p.K = K; - p.stride = stride; - p.p0 = p0; + uint32_t ncols = src0->ne[0]; + uint32_t nrows = ggml_nrows(src0); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_COL2IM_1D, std::move(p)); -} + uint32_t ncols_pad_log2 = (uint32_t)ceilf(log2f(float(ncols))); + uint32_t ncolsp2 = 1 << ncols_pad_log2; -// Dispatch the fused snake activation: y = x + sin^2(a * x) * inv_b. -// Match the naive mul -> sin -> sqr -> mul -> add chain and run the -// dedicated kernel directly. The pattern is validated by -// ggml_vk_can_fuse_snake before this call. -static void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) { - const ggml_tensor * mul0 = cgraph->nodes[node_idx + 0]; - const ggml_tensor * sqr = cgraph->nodes[node_idx + 2]; - const ggml_tensor * mul1 = cgraph->nodes[node_idx + 3]; - ggml_tensor * add = cgraph->nodes[node_idx + 4]; + vk_op_argsort_push_constants pc { ncols, ncolsp2, ncols_pad_log2, nrows, op_params[0], 0, 0, 0, 0, }; - // x carries the full activation shape, a is the broadcast operand - const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; - const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; + // Pick the largest workgroup size <= ncolsp2 + uint32_t pipeline_idx = std::min(ncols_pad_log2, num_argsort_pipelines - 1); - // mul1 reads sqr and inv_b in either operand order - const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; + // Use the "small" argsort shader if the whole sort can be done by a single workgroup. + bool use_small = ncols_pad_log2 <= ctx->device->max_workgroup_size_log2 && + ctx->device->pipeline_argsort_f32[pipeline_idx] != nullptr; - vk_pipeline pipeline = nullptr; - switch (x->type) { - case GGML_TYPE_F32: pipeline = ctx->device->pipeline_snake_f32; break; - case GGML_TYPE_F16: pipeline = ctx->device->pipeline_snake_f16; break; - case GGML_TYPE_BF16: pipeline = ctx->device->pipeline_snake_bf16; break; - default: GGML_ABORT("unsupported type"); + vk_pipeline pipeline = use_small ? ctx->device->pipeline_argsort_f32[pipeline_idx] + : ctx->device->pipeline_argsort_large_f32[pipeline_idx]; + + vk_subbuffer src0_buf = ggml_vk_tensor_subbuffer(ctx, src0); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); + vk_subbuffer subbuf1 = dst_buf; + + // Reserve space for ivec2 per element, with rows padded to a power of two + if (!use_small) { + const size_t x_sz = size_t{ncolsp2} * nrows * 2 * sizeof(int); + + if (ctx->prealloc_size_x < x_sz) { + ctx->prealloc_size_x = x_sz; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + subbuf1 = { ctx->prealloc_x, 0, ctx->prealloc_x->size }; } - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - vk_subbuffer x_buf = ggml_vk_tensor_subbuffer(ctx, x); - vk_subbuffer a_buf = ggml_vk_tensor_subbuffer(ctx, a); - vk_subbuffer inv_b_buf = ggml_vk_tensor_subbuffer(ctx, inv_b); - vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, add); + std::array<uint32_t, 3> elements; - vk_op_snake_push_constants pc{}; - pc.ne0 = static_cast<uint32_t>(x->ne[0]); - pc.ne1 = static_cast<uint32_t>(x->ne[1]); + elements[0] = ncolsp2; + elements[1] = std::min((uint32_t)ggml_nrows(src0), ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + elements[2] = 1; - std::array<uint32_t, 3> elements = { pc.ne0, pc.ne1, 1 }; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, a_buf, inv_b_buf, dst_buf }, pc, elements); + // First dispatch initializes tmp_idx and does the first N passes where + // there is only communication between threads in the same workgroup. + { + vk_op_argsort_push_constants pc2 = pc; + pc2.outer_start = 0; + pc2.outer_end = std::min(ncols_pad_log2, ctx->device->max_workgroup_size_log2); + pc2.inner_start = 0; + pc2.inner_end = 100; + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, subbuf1, dst_buf }, pc2, elements); + } + if (!use_small) { + ggml_vk_sync_buffers(ctx, subctx); + // Loop over outer/inner passes, synchronizing between each pass. + for (uint32_t outer = ctx->device->max_workgroup_size_log2; outer < ncols_pad_log2; ++outer) { + for (uint32_t inner = 0; inner < outer + 1; ++inner) { + vk_op_argsort_push_constants pc2 = pc; + pc2.outer_start = outer; + pc2.outer_end = outer + 1; + pc2.inner_start = inner; + pc2.inner_end = inner + 1; + // When the inner idx is large enough, there's only communication + // within a workgroup. So the remaining inner iterations can all + // run in the same dispatch. + if (outer - inner < pipeline_idx) { + pc2.inner_end = 100; + inner = outer; + pipeline = ctx->device->pipeline_argsort_large_f32[pipeline_idx]; + } else { + // Smaller workgroup empirically seems to perform better + pipeline = ctx->device->pipeline_argsort_large_f32[pipeline_idx - 2]; + } + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, subbuf1, dst_buf }, pc2, elements); + ggml_vk_sync_buffers(ctx, subctx); + } + } + ctx->prealloc_x_need_sync = true; + } } -static void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - uint32_t op = static_cast<uint32_t>(dst->op_params[0]); - const int32_t k0 = dst->op_params[1]; - const int32_t s0 = dst->op_params[2]; - const int32_t p0 = dst->op_params[3]; +void ggml_vk_topk(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + uint32_t ncols = src0->ne[0]; + uint32_t nrows = ggml_nrows(src0); + uint32_t k = dst->ne[0]; - const uint32_t IL = src0->ne[0]; + // tournament path is faster where it fits; use radix-select only past its k limit + const uint32_t k_min_pipeline = std::max((uint32_t) log2f(float(k)) + 1, ctx->device->subgroup_size_log2); + if (k_min_pipeline >= num_topk_pipelines || ctx->device->pipeline_topk_f32[k_min_pipeline] == nullptr) { + vk_pipeline pipeline = ctx->device->pipeline_topk_radix_f32; + GGML_ASSERT(pipeline != nullptr); - const uint32_t N = dst->ne[3] * dst->ne[2]; + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } - const uint32_t OC = dst->ne[1]; - const uint32_t OL = dst->ne[0]; + vk_op_topk_radix_push_constants pc { ncols, k, nrows, 0, 0, 0 }; + std::array<uint32_t, 3> elements { + pipeline->wg_denoms[0], + std::min(nrows, ctx->device->properties.limits.maxComputeWorkGroupCount[1]), + 1, + }; + // the non-QSA path only uses bindings 0/1; bind valid buffers for the unused QSA slots + vk_subbuffer src0_buf = ggml_vk_tensor_subbuffer(ctx, src0); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { src0_buf, dst_buf, src0_buf, src0_buf, src0_buf }, pc, elements); + return; + } - const uint32_t parallel_elements = N * OC * OL; + vk_op_topk_push_constants pc { ncols, ncols, ncols, k, nrows, 0, 0 }; - ggml_vk_op_f32<vk_op_pool1d_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_POOL_1D, { - IL, OL, OC, - parallel_elements, - op, - k0, s0, p0, - }); -} + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } -static void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - uint32_t op = static_cast<uint32_t>(dst->op_params[0]); - const int32_t k1 = dst->op_params[1]; - const int32_t k0 = dst->op_params[2]; - const int32_t s1 = dst->op_params[3]; - const int32_t s0 = dst->op_params[4]; - const int32_t p1 = dst->op_params[5]; - const int32_t p0 = dst->op_params[6]; + std::array<uint32_t, 3> elements; + elements[1] = std::min(nrows, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + elements[2] = 1; - const uint32_t IH = src0->ne[1]; - const uint32_t IW = src0->ne[0]; + uint32_t num_elements = ncols; - const uint32_t N = dst->ne[3]; + // Each iteration reduces a workgroup's worth of elements down to the K + // largest elements. Repeat until we have the top K elements. + // Need to do at least one iteration to write out the results. + bool done_one_iter = false; + uint32_t dbl_buf_index = 0; + size_t dbl_buf_size; + while (num_elements > k || !done_one_iter) { - const uint32_t OC = dst->ne[2]; - const uint32_t OH = dst->ne[1]; - const uint32_t OW = dst->ne[0]; + // Prefer going as small as num_topk_pipelines - 3 for perf reasons. + // But if K is larger, then we need a larger workgroup + uint32_t max_pipeline = num_topk_pipelines - 1; + uint32_t preferred_pipeline = std::max(num_topk_pipelines - 3, (uint32_t)log2f(float(k)) + 2); + max_pipeline = std::min(preferred_pipeline, max_pipeline); + uint32_t min_pipeline = (uint32_t)log2f(float(k)) + 1; + // require full subgroup + min_pipeline = std::max(min_pipeline, ctx->device->subgroup_size_log2); - const uint32_t parallel_elements = N * OC * OH * OW; + uint32_t pipeline_idx = (uint32_t)ceilf(log2f(float(num_elements))); + pipeline_idx = std::min(pipeline_idx, max_pipeline); + pipeline_idx = std::max(pipeline_idx, min_pipeline); - ggml_vk_op_f32<vk_op_pool2d_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_POOL_2D, { - IW, IH, OW, OH, OC, - parallel_elements, - op, - k0, k1, s0, s1, p0, p1, - }); -} + if (num_elements > (1u << pipeline_idx)) { + // If we could finish on this loop iteration (i.e. a single workgroup) + // then do so. It's better than the overhead of another pass. + for (uint32_t i = pipeline_idx; i < num_topk_pipelines; ++i) { + if (num_elements <= (1u << i)) { + pipeline_idx = i; + break; + } + } + } -static void ggml_vk_conv_2d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0, - const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); + vk_pipeline pipeline = ctx->device->pipeline_topk_f32[pipeline_idx]; + // If the device doesn't support a pipeline this large, use smaller + while (!pipeline) { + pipeline_idx--; + GGML_ASSERT(pipeline_idx >= min_pipeline); + pipeline = ctx->device->pipeline_topk_f32[pipeline_idx]; + } - GGML_TENSOR_BINARY_OP_LOCALS - GGML_ASSERT(nb00 == sizeof(float) || nb00 == sizeof(ggml_fp16_t)); - GGML_ASSERT(nb10 == sizeof(float)); - GGML_ASSERT(nb0 == sizeof(float)); + vk_op_topk_push_constants pc2 = pc; + pc2.ncols_input = num_elements; - bool transpose = dst->op == GGML_OP_CONV_TRANSPOSE_2D; + // Number of elements remaining after this pass + uint32_t num_dst_elements = (num_elements / pipeline->wg_denoms[0]) * k + std::min(k, num_elements % pipeline->wg_denoms[0]); - vk_op_conv2d_push_constants p{}; - p.Cout = static_cast<uint32_t>(!transpose ? ne03 : ne02); - p.Cin = static_cast<uint32_t>(!transpose ? ne02 : ne03); - p.N = static_cast<uint32_t>(ne13); - GGML_ASSERT(p.Cout == ne2); - GGML_ASSERT(p.Cin == ne12); + pc2.ncols_output = num_dst_elements; - p.W = static_cast<uint32_t>(ne10); - p.H = static_cast<uint32_t>(ne11); - p.OW = static_cast<uint32_t>(ne0); - p.OH = static_cast<uint32_t>(ne1); + if (!done_one_iter) { + // Reserve space for ivec2 per element, double buffered + // K per workgroup per row + dbl_buf_size = num_dst_elements * nrows * 2 * sizeof(int); + dbl_buf_size = ROUNDUP_POW2(dbl_buf_size, ctx->device->properties.limits.minStorageBufferOffsetAlignment); + const size_t x_sz = dbl_buf_size * 2; - p.nb01 = static_cast<uint32_t>(nb01 / nb00); - p.nb02 = static_cast<uint32_t>(nb02 / nb00); - p.nb03 = static_cast<uint32_t>(nb03 / nb00); + if (ctx->prealloc_size_x < x_sz) { + ctx->prealloc_size_x = x_sz; + ggml_vk_preallocate_buffers(ctx, subctx); + } + } - p.nb11 = static_cast<uint32_t>(nb11 / nb10); - p.nb12 = static_cast<uint32_t>(nb12 / nb10); - p.nb13 = static_cast<uint32_t>(nb13 / nb10); + vk_subbuffer src_buf; + vk_subbuffer dst_buf; + + if (num_elements == ncols) { + pc2.first_pass = 1; + src_buf = ggml_vk_tensor_subbuffer(ctx, src0); + } else { + src_buf = { ctx->prealloc_x, dbl_buf_index * dbl_buf_size, dbl_buf_size }; + } + if (num_dst_elements == k) { + pc2.last_pass = 1; + dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); + } else { + dst_buf = { ctx->prealloc_x, (dbl_buf_index ^ 1) * dbl_buf_size, dbl_buf_size }; + } - p.nb1 = static_cast<uint32_t>(nb1 / nb0); - p.nb2 = static_cast<uint32_t>(nb2 / nb0); - p.nb3 = static_cast<uint32_t>(nb3 / nb0); + elements[0] = num_elements; - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, dst->op, std::move(p)); + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src_buf, dst_buf }, pc2, elements); + num_elements = num_dst_elements; + dbl_buf_index ^= 1; + if (num_elements > k) { + ggml_vk_sync_buffers(ctx, subctx); + } + done_one_iter = true; + } + ctx->prealloc_x_need_sync = true; } -static void ggml_vk_conv_3d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0, - const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); - - GGML_TENSOR_BINARY_OP_LOCALS - GGML_ASSERT(nb00 == sizeof(float) || nb00 == sizeof(ggml_fp16_t)); - GGML_ASSERT(nb10 == sizeof(float)); - GGML_ASSERT(nb0 == sizeof(float)); - - vk_op_conv3d_push_constants p{}; - p.IC = static_cast<uint32_t>(ggml_get_op_params_i32(dst, 9)); - p.N = static_cast<uint32_t>(ggml_get_op_params_i32(dst, 10)); - p.OC = static_cast<uint32_t>(ggml_get_op_params_i32(dst, 11)); - GGML_ASSERT(src0->ne[3] == (int64_t)p.IC * p.OC); - GGML_ASSERT(src1->ne[3] == (int64_t)p.IC * p.N); - GGML_ASSERT(dst->ne[3] == (int64_t)p.OC * p.N); +void ggml_vk_topk_qsa(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx) { + const ggml_tensor * get_rows = cgraph->nodes[node_idx + 0]; + const ggml_tensor * add = cgraph->nodes[node_idx + ctx->num_additional_fused_ops - 1]; + ggml_tensor * top_k = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; - p.IW = static_cast<uint32_t>(ne10); - p.IH = static_cast<uint32_t>(ne11); - p.ID = static_cast<uint32_t>(ne12); - p.OW = static_cast<uint32_t>(ne0); - p.OH = static_cast<uint32_t>(ne1); - p.OD = static_cast<uint32_t>(ne2); + const ggml_tensor * scores = get_rows->src[0]; // [n_tps, n_blocks, n_stream] + const ggml_tensor * cell_blk = get_rows->src[1]; // [n_kv, n_stream] - // the shader clamps src addresses to p.IC * p.N * p.IW * p.IH * p.ID - 1 in uint32, so the - // total input element count must fit in a uint32. - GGML_ASSERT((uint64_t)p.IC * p.N * p.IW * p.IH * p.ID <= 0xFFFFFFFFull); + // raw f16 mask: follow the reshape/cpy chain back to the materialized input + const ggml_tensor * mask = add->src[1]; + while (mask->op == GGML_OP_RESHAPE || mask->op == GGML_OP_CPY) { + mask = mask->src[0]; + } - p.nb01 = static_cast<uint32_t>(nb01 / nb00); - p.nb02 = static_cast<uint32_t>(nb02 / nb00); - p.nb03 = static_cast<uint32_t>(nb03 / nb00); + const uint32_t n_tps = scores->ne[0]; + const uint32_t n_blocks = scores->ne[1]; + const uint32_t n_stream = scores->ne[2]; + const uint32_t n_kv = cell_blk->ne[0]; + const uint32_t width = top_k->ne[0]; + const uint32_t nrows = n_tps * n_stream; - p.nb11 = static_cast<uint32_t>(nb11 / nb10); - p.nb12 = static_cast<uint32_t>(nb12 / nb10); - p.nb13 = static_cast<uint32_t>(nb13 / nb10); + vk_pipeline pipeline = ctx->device->pipeline_topk_radix_qsa; + GGML_ASSERT(pipeline != nullptr); - p.nb1 = static_cast<uint32_t>(nb1 / nb0); - p.nb2 = static_cast<uint32_t>(nb2 / nb0); - p.nb3 = static_cast<uint32_t>(nb3 / nb0); + // scratch holds the gathered+masked input, materialized once and reused across passes + const size_t scratch_size = size_t{ n_kv } * nrows * sizeof(float); + if (ctx->prealloc_size_x < scratch_size) { + ctx->prealloc_size_x = scratch_size; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_3D, std::move(p)); + vk_op_topk_radix_push_constants pc { n_kv, width, nrows, n_tps, n_blocks, n_stream }; + std::array<uint32_t, 3> elements { + pipeline->wg_denoms[0], + std::min(nrows, ctx->device->properties.limits.maxComputeWorkGroupCount[1]), + 1, + }; + vk_subbuffer scratch_buf { ctx->prealloc_x, 0, ctx->prealloc_x->size }; + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { ggml_vk_tensor_subbuffer(ctx, scores), ggml_vk_tensor_subbuffer(ctx, top_k), + ggml_vk_tensor_subbuffer(ctx, cell_blk), ggml_vk_tensor_subbuffer(ctx, mask), + scratch_buf }, pc, elements); + ctx->prealloc_x_need_sync = true; } -static void ggml_vk_conv_2d_dw(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - vk_op_conv2d_dw_push_constants p{}; - p.ne = ggml_nelements(dst); - p.channels = dst->ne[2]; - p.batches = dst->ne[3]; - p.dst_w = dst->ne[0]; - p.dst_h = dst->ne[1]; - p.src_w = src1->ne[0]; - p.src_h = src1->ne[1]; - p.knl_w = src0->ne[0]; - p.knl_h = src0->ne[1]; - p.stride_x = dst->op_params[0]; - p.stride_y = dst->op_params[1]; - p.pad_x = dst->op_params[2]; - p.pad_y = dst->op_params[3]; - p.dilation_x = dst->op_params[4]; - p.dilation_y = dst->op_params[5]; - - GGML_ASSERT(src0->ne[3] == p.channels); - GGML_ASSERT(src1->ne[3] == p.batches); - - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_2D_DW, std::move(p)); +void ggml_vk_sum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, ggml_nelements(src0)); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SUM, p); } -static void ggml_vk_leaky_relu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - const float * op_params = (const float *)dst->op_params; - vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); - p.param1 = op_params[0]; +void ggml_vk_sum_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SUM_ROWS, p); +} - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LEAKY_RELU, std::move(p)); +void ggml_vk_mean(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]); + p.weight = 1.0f / (float)src0->ne[0]; + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_MEAN, p); } -#ifdef GGML_VULKAN_RUN_TESTS -static void ggml_vk_print_matrix_area(const void * data, ggml_type type, int ne0, int ne1, int i0, int i1, int i2) { - if (type != GGML_TYPE_F32 && type != GGML_TYPE_F16) { +void ggml_vk_cumsum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_sum_rows_push_constants pc = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]); + // Use the single pass shader when the rows are small or there are enough rows to fill the GPU. + // For fewer, larger rows, use the multipass shader to spread each row across SMs. + if (dst->ne[0] <= 4096 || ggml_nrows(dst) >= ctx->device->shader_core_count) { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CUMSUM, pc); return; } - i0 = std::max(i0, 5); - i1 = std::max(i1, 5); - i2 = std::max(i2, 0); - fprintf(stderr, " "); - for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { - fprintf(stderr, "%7d ", idx1); - } - fprintf(stderr, "\n"); - for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) { - fprintf(stderr, "%7d: ", idx0); - for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { - if (idx0 >= 0 && idx0 < ne0 && idx1 >= 0 && idx1 < ne1) { - float val; - if (type == GGML_TYPE_F32) { - val = *((const float *) data + i2*ne1*ne0 + idx1*ne0 + idx0); - } else if (type == GGML_TYPE_F16) { - val = ggml_fp16_to_fp32(*((const ggml_fp16_t *) data + i2*ne1*ne0 + idx1*ne0 + idx0)); - } else { - GGML_ABORT("fatal error"); - } - fprintf(stderr, "% 7.2f ", val); - } else { - fprintf(stderr, " "); - } - } - fprintf(stderr, "\n"); - } -} - -template <typename X_TYPE, typename Y_TYPE> -static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, int split_k, int shader_size) { - VK_LOG_DEBUG("ggml_vk_test_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << shader_size << ")"); - const size_t x_ne = m * k * batch; - const size_t y_ne = k * n * batch; - const size_t d_ne = m * n * batch; - - vk_pipeline p; - std::string shname; - if (shader_size == 0) { - if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32->a_s; - shname = "F32_ALIGNED_S"; - } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32_f16->a_s; - shname = "F32_F16_ALIGNED_S"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_s; - shname = "F16_F32_ALIGNED_S"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16.f32acc->a_s; - shname = "F16_ALIGNED_S"; - } else { - GGML_ABORT("fatal error"); - } - } else if (shader_size == 1) { - if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32->a_m; - shname = "F32_ALIGNED_M"; - } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32_f16->a_m; - shname = "F32_F16_ALIGNED_M"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_m; - shname = "F16_F32_ALIGNED_M"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16.f32acc->a_m; - shname = "F16_ALIGNED_M"; - } else { - GGML_ABORT("fatal error"); - } - } else if (shader_size == 2) { - if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32->a_l; - shname = "F32_ALIGNED_L"; - } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32_f16->a_l; - shname = "F32_F16_ALIGNED_L"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_l; - shname = "F16_F32_ALIGNED_L"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16.f32acc->a_l; - shname = "F16_ALIGNED_L"; - } else { - GGML_ABORT("fatal error"); - } - } else { - GGML_ASSERT(0); - } - const size_t kpad = ggml_vk_align_size(k, p->align); - - if (k != kpad) { - if (shader_size == 0) { - if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32->s; - shname = "F32_S"; - } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32_f16->s; - shname = "F32_F16_S"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->s; - shname = "F16_F32_S"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16.f32acc->s; - shname = "F16_S"; - } - } else if (shader_size == 1) { - if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32->m; - shname = "F32_M"; - } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32_f16->m; - shname = "F32_F16_M"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->m; - shname = "F16_F32_M"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16.f32acc->m; - shname = "F16_M"; - } - } else if (shader_size == 2) { - if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32->l; - shname = "F32_L"; - } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f32_f16->l; - shname = "F32_F16_L"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->l; - shname = "F16_F32_L"; - } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) { - p = ctx->device->pipeline_matmul_f16.f32acc->l; - shname = "F16_L"; - } - } - } + // First pass computes partial sums within a block, and stores the last partial + // to the temp buffer. Second pass sums the block partials from the temp buffer + // and adds that to the result of the first pass. + vk_pipeline pipeline1 = ctx->device->pipeline_cumsum_multipass1_f32; + vk_pipeline pipeline2 = ctx->device->pipeline_cumsum_multipass2_f32; + GGML_ASSERT(pipeline1 != nullptr && pipeline2 != nullptr); + + ggml_pipeline_request_descriptor_sets(ctx, pipeline1, 1); + ggml_pipeline_request_descriptor_sets(ctx, pipeline2, 1); - if (split_k > 1) { - ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it); + std::array<uint32_t, 3> elements; - if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) { - // Resize buffer - if (ctx->prealloc_split_k != nullptr) { - ggml_vk_destroy_buffer(ctx->prealloc_split_k); - } - ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal}); - } - } + elements[0] = dst->ne[0]; + elements[1] = (uint32_t)ggml_nrows(dst); + elements[2] = 1; - ggml_pipeline_allocate_descriptor_sets(ctx); + size_t temp_size = sizeof(float) * elements[0] * ggml_nrows(dst); - vk_buffer d_X = ggml_vk_create_buffer_check(ctx->device, sizeof(X_TYPE) * x_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal}); - vk_buffer d_Y = ggml_vk_create_buffer_check(ctx->device, sizeof(Y_TYPE) * y_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal}); - vk_buffer d_D = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal}); - - X_TYPE* x = (X_TYPE *) malloc(sizeof(X_TYPE) * x_ne); - Y_TYPE* y = (Y_TYPE *) malloc(sizeof(Y_TYPE) * y_ne); - float* d = (float *) malloc(sizeof(float) * d_ne); - - for (size_t i = 0; i < x_ne; i++) { - if (std::is_same<float, X_TYPE>()) { - x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; - // x[i] = 1.0f; - // x[i] = i + 1; - // x[i] = (i % k == i / k) ? 1.0f : 0.0f; - } else if (std::is_same<ggml_fp16_t, X_TYPE>()) { - x[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f); - // x[i] = ggml_fp32_to_fp16(1.0f); - // x[i] = ggml_fp32_to_fp16(i + 1); - // x[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f); - } else { - GGML_ABORT("fatal error"); - } + if (ctx->prealloc_size_split_k < temp_size) { + ctx->prealloc_size_split_k = temp_size; + ggml_vk_preallocate_buffers(ctx, subctx); } - for (size_t i = 0; i < y_ne; i++) { - if (std::is_same<float, Y_TYPE>()) { - y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; - // y[i] = (i % k == i / k) ? 1.0f : 0.0f; - // y[i] = i + 1; - } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) { - y[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f); - // y[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f); - // y[i] = ggml_fp32_to_fp16(i + 1); - } else { - GGML_ABORT("fatal error"); - } + + vk_subbuffer src_buf = ggml_vk_tensor_subbuffer(ctx, src0); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); + vk_subbuffer temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_split_k, 0); + + if (ctx->prealloc_split_k_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); } - ggml_vk_buffer_write(d_X, 0, x, sizeof(X_TYPE) * k * m * batch); - ggml_vk_buffer_write(d_Y, 0, y, sizeof(Y_TYPE) * k * n * batch); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline1, {src_buf, dst_buf, temp_buf}, pc, elements); + ggml_vk_sync_buffers(ctx, subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline2, {src_buf, dst_buf, temp_buf}, pc, elements); + + ctx->prealloc_split_k_need_sync = true; +} - vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool); - ggml_vk_ctx_begin(ctx->device, subctx); - for (size_t i = 0; i < num_it; i++) { - ggml_vk_matmul( - ctx, subctx, p, ggml_vk_subbuffer(ctx, d_X), ggml_vk_subbuffer(ctx, d_Y), ggml_vk_subbuffer(ctx, d_D), ggml_vk_subbuffer(ctx, ctx->prealloc_split_k), - m, n, k, - k, k, m, k*m, k*n, m*n, - split_k, batch, batch, batch, 1, 1, n - ); +static std::array<uint32_t, 3> ggml_vk_nrows_elements(uint32_t nr) { + if (nr > 262144) { + return { 512, 512, CEIL_DIV(nr, 262144) }; } - ggml_vk_ctx_end(subctx); + if (nr > 512) { + return { 512, CEIL_DIV(nr, 512), 1 }; + } + return { nr, 1, 1 }; +} - auto begin = std::chrono::high_resolution_clock::now(); - ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences", ctx->device); - ctx->device->device.resetFences({ ctx->fence }); - ggml_vk_queue_command_pools_cleanup(ctx->device); +void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; - auto end = std::chrono::high_resolution_clock::now(); - double time = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0; + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(src1)); + GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_are_same_shape(src0, src1)); + GGML_ASSERT(ggml_is_scalar(dst)); - // copy dst to host - ggml_vk_buffer_read(d_D, 0, d, sizeof(float) * d_ne); + const uint32_t nclasses = (uint32_t)src0->ne[0]; + const uint32_t nrows = (uint32_t)ggml_nrows(src0); - float * d_chk = (float *) malloc(sizeof(float) * d_ne); + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, nullptr, dst, GGML_OP_CROSS_ENTROPY_LOSS); + GGML_ASSERT(pipeline != nullptr); - ggml_init_params iparams = { - /*.mem_size =*/ 1024*1024*1024, - /*.mem_buffer =*/ NULL, - /*.no_alloc =*/ true, - }; + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_sum_rows_f32, 1); - ggml_context * ggml_ctx = ggml_init(iparams); + vk_subbuffer src0_buf = ggml_vk_tensor_subbuffer(ctx, src0); + vk_subbuffer src1_buf = ggml_vk_tensor_subbuffer(ctx, src1); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); - ggml_type src0_type; - ggml_type src1_type; + const vk_op_push_constants pc = { nclasses, nrows, 0.0f, 0.0f, 0.0f, 0.0f }; - if (std::is_same<float, X_TYPE>()) { - src0_type = GGML_TYPE_F32; - } else if (std::is_same<ggml_fp16_t, X_TYPE>()) { - src0_type = GGML_TYPE_F16; - } else { - GGML_ABORT("fatal error"); + const size_t tmp_size = (size_t)nrows * sizeof(float); + if (ctx->prealloc_size_x < tmp_size) { + ctx->prealloc_size_x = tmp_size; + ggml_vk_preallocate_buffers(ctx, subctx); } - if (std::is_same<float, Y_TYPE>()) { - src1_type = GGML_TYPE_F32; - } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) { - src1_type = GGML_TYPE_F16; - } else { - GGML_ABORT("fatal error"); + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); } - ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, src0_type, k, m, batch); - ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, src1_type, k, n, batch); - ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml); + vk_subbuffer tmp_buf = { ctx->prealloc_x, 0, tmp_size }; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, tmp_buf }, pc, ggml_vk_nrows_elements(nrows)); + ggml_vk_sync_buffers(ctx, subctx); - src0_ggml->data = x; - src1_ggml->data = y; - tensor_ggml->data = d_chk; + vk_op_sum_rows_push_constants sp = {}; + sp.n_cols = nrows; + sp.ne01 = 1; + sp.ne02 = 1; + sp.weight = 1.0f; + init_pushconst_fastdiv(sp); + sp.misalign_offsets = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); - ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx); - ggml_build_forward_expand(cgraph, tensor_ggml); + ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_sum_rows_f32, { tmp_buf, dst_buf }, sp, { 1, 1, 1 }); + ctx->prealloc_x_need_sync = true; +} - ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1); +void ggml_vk_cross_entropy_loss_back(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const ggml_tensor * grad = dst->src[0]; + const ggml_tensor * logits = dst->src[1]; + const ggml_tensor * labels = dst->src[2]; - ggml_free(ggml_ctx); + GGML_ASSERT(grad->type == GGML_TYPE_F32); + GGML_ASSERT(logits->type == GGML_TYPE_F32); + GGML_ASSERT(labels->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_scalar(grad)); + GGML_ASSERT(ggml_is_contiguous(grad)); + GGML_ASSERT(ggml_is_contiguous(logits)); + GGML_ASSERT(ggml_is_contiguous(labels)); + GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_are_same_shape(logits, labels)); + GGML_ASSERT(ggml_are_same_shape(logits, dst)); - double avg_err = 0.0; - int first_err_n = -1; - int first_err_m = -1; - int first_err_b = -1; + const uint32_t nclasses = (uint32_t)logits->ne[0]; + const uint32_t nrows = (uint32_t)ggml_nrows(logits); - for (size_t i = 0; i < m*n*batch; i++) { - double err = std::fabs(d[i] - d_chk[i]); - avg_err += err; + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, grad, logits, labels, dst, GGML_OP_CROSS_ENTROPY_LOSS_BACK); + GGML_ASSERT(pipeline != nullptr); - if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) { - first_err_b = i / (m * n); - first_err_n = (i % (m * n)) / m; - first_err_m = (i % (m * n)) % m; - } - } + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - avg_err /= m * n; + vk_subbuffer grad_buf = ggml_vk_tensor_subbuffer(ctx, grad); + vk_subbuffer logits_buf = ggml_vk_tensor_subbuffer(ctx, logits); + vk_subbuffer labels_buf = ggml_vk_tensor_subbuffer(ctx, labels); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); - double tflops = 2.0*m*n*k*batch*num_it / (time / 1000.0) / (1000.0*1000.0*1000.0*1000.0); + const vk_op_push_constants pc = { nclasses, nrows, 0.0f, 0.0f, 0.0f, 0.0f }; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { grad_buf, logits_buf, labels_buf, dst_buf }, pc, ggml_vk_nrows_elements(nrows)); +} - std::cerr << "TEST " << shname << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl; +void ggml_vk_argmax(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_ARGMAX, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], 0.0f, 0.0f, 0.0f, 0.0f }); +} - if (avg_err > 0.1 || std::isnan(avg_err)) { - std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl; - std::cerr << "Actual result: " << std::endl << std::endl; - ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); - std::cerr << "Expected result: " << std::endl << std::endl; - ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); +void ggml_vk_count_equal(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_COUNT_EQUAL, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f, 0.0f, 0.0f }); +} - if (split_k > 1) { - float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k); - ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k); +void ggml_vk_solve_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); - std::cerr << "d_buf0: " << std::endl << std::endl; - ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SOLVE_TRI, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, 0, + }); +} - std::cerr << "d_buf1: " << std::endl << std::endl; - ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); +void ggml_vk_im2col(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const int32_t s0 = dst->op_params[0]; + const int32_t s1 = dst->op_params[1]; + const int32_t p0 = dst->op_params[2]; + const int32_t p1 = dst->op_params[3]; + const int32_t d0 = dst->op_params[4]; + const int32_t d1 = dst->op_params[5]; - std::cerr << "d_buf2: " << std::endl << std::endl; - ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + const bool is_2D = dst->op_params[6] == 1; - std::cerr << "d_buf3: " << std::endl << std::endl; - ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + const uint32_t IC = src1->ne[is_2D ? 2 : 1]; + const uint32_t IH = is_2D ? src1->ne[1] : 1; + const uint32_t IW = src1->ne[0]; - free(split_k_buf); - } - } + const uint32_t KH = is_2D ? src0->ne[1] : 1; + const uint32_t KW = src0->ne[0]; - free(d_chk); + const uint32_t OH = is_2D ? dst->ne[2] : 1; + const uint32_t OW = dst->ne[1]; - ggml_vk_command_pool_cleanup(ctx->device, ctx->compute_cmd_pool); + const uint32_t offset_delta = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32 + const uint32_t batch_offset = src1->nb[is_2D ? 3 : 2] / 4; // nb is byte offset, src is type float32 - ggml_vk_destroy_buffer(d_X); - ggml_vk_destroy_buffer(d_Y); - ggml_vk_destroy_buffer(d_D); + const uint32_t batch = src1->ne[is_2D ? 3 : 2]; - free(x); - free(y); - free(d); -} + const ggml_backend_vk_buffer_context * d_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; + const vk_buffer d_buf = d_buf_ctx->dev_buffer; -static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, int i0, int i1, int i2, int i3) { - if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16) { - return; - } - i0 = std::max(i0, 5); - i1 = std::max(i1, 5); - i2 = std::max(i2, 0); - i3 = std::max(i3, 0); - fprintf(stderr, " "); - for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { - fprintf(stderr, "%7d ", idx1); - } - fprintf(stderr, "\n"); - for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) { - fprintf(stderr, "%7d: ", idx0); - for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { - if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) { - float val; - if (tensor->type == GGML_TYPE_F32) { - val = *(float *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]); - } else if (tensor->type == GGML_TYPE_F16) { - val = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0])); - } else { - GGML_ABORT("fatal error"); - } - fprintf(stderr, "% 7.2f ", val); - } else { - fprintf(stderr, " "); - } - } - fprintf(stderr, "\n"); - } -} + const vk::DeviceAddress dst_addr = d_buf->bda_addr + vk_tensor_offset(dst) + dst->view_offs; -static void ggml_vk_quantize_data(const float * from, void * to, size_t ne, ggml_type quant) { - ggml_quantize_chunk(quant, from, to, 0, 1, ne, nullptr); + ggml_vk_op_f32<vk_op_im2col_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_IM2COL, { + dst_addr, + batch_offset, offset_delta, + IC, IW, IH, OW, OH, KW, KH, + OH * batch, + IC * KH * KW, + s0, s1, p0, p1, d0, d1, batch * IC + }); } -static void ggml_vk_dequantize_data(const void * from, float * to, size_t ne, ggml_type quant) { - if (quant == GGML_TYPE_F32) { - memcpy(to, from, sizeof(float) * ne); - return; - } +void ggml_vk_im2col_3d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS - const auto * tt = ggml_get_type_traits(quant); + const int32_t s0 = ((const int32_t *)(dst->op_params))[0]; + const int32_t s1 = ((const int32_t *)(dst->op_params))[1]; + const int32_t s2 = ((const int32_t *)(dst->op_params))[2]; + const int32_t p0 = ((const int32_t *)(dst->op_params))[3]; + const int32_t p1 = ((const int32_t *)(dst->op_params))[4]; + const int32_t p2 = ((const int32_t *)(dst->op_params))[5]; + const int32_t d0 = ((const int32_t *)(dst->op_params))[6]; + const int32_t d1 = ((const int32_t *)(dst->op_params))[7]; + const int32_t d2 = ((const int32_t *)(dst->op_params))[8]; + const int32_t IC = ((const int32_t *)(dst->op_params))[9]; - ggml_to_float_t dequant_fn = tt->to_float; + const int64_t N = ne13 / IC; + const int64_t ID = ne12; + const int64_t IH = ne11; + const int64_t IW = ne10; - dequant_fn(from, to, ne); -} + const int64_t KD = ne02; + const int64_t KH = ne01; + const int64_t KW = ne00; -static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) { - VK_LOG_DEBUG("ggml_vk_test_dequant(" << ne << ")"); - const size_t x_sz = sizeof(float) * ne; - const size_t x_sz_f16 = sizeof(ggml_fp16_t) * ne; - const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant); - float * x = (float *) malloc(x_sz); - void * qx = malloc(qx_sz); - vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); - vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz_f16, {vk::MemoryPropertyFlagBits::eDeviceLocal}); - float * x_ref = (float *) malloc(x_sz); - ggml_fp16_t * x_chk = (ggml_fp16_t *) malloc(x_sz_f16); + const int64_t OD = ne3 / N; + const int64_t OH = ne2; + const int64_t OW = ne1; - for (size_t i = 0; i < ne; i++) { - x[i] = rand() / (float)RAND_MAX; - } + const ggml_backend_vk_buffer_context * d_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; + const vk_buffer d_buf = d_buf_ctx->dev_buffer; - vk_pipeline p = ggml_vk_get_to_fp16(ctx, quant); + const vk::DeviceAddress dst_addr = d_buf->bda_addr + vk_tensor_offset(dst) + dst->view_offs; - ggml_vk_quantize_data(x, qx, ne, quant); - ggml_vk_dequantize_data(qx, x_ref, ne, quant); + vk_op_im2col_3d_push_constants pc {}; - ggml_pipeline_request_descriptor_sets(ctx, p, 1); + pc.dst_addr = dst_addr; + pc.nb10 = nb10 / ggml_type_size(src1->type); + pc.nb11 = nb11 / ggml_type_size(src1->type); + pc.nb12 = nb12 / ggml_type_size(src1->type); + pc.nb13 = nb13 / ggml_type_size(src1->type); + pc.s0 = s0; + pc.s1 = s1; + pc.s2 = s2; + pc.p0 = p0; + pc.p1 = p1; + pc.p2 = p2; + pc.d0 = d0; + pc.d1 = d1; + pc.d2 = d2; + pc.IW = IW; + pc.IH = IH; + pc.ID = ID; + pc.IC = IC; + pc.KW = KW; + pc.OH = OH; + pc.KD_KH_KW = KD*KH*KW; + pc.KH_KW = KH*KW; + pc.IC_KD_KH_KW = IC*KD*KH*KW; + pc.N_OD_OH = N*OD*OH; + pc.OD_OH = OD*OH; + pc.OD_OH_OW_IC_KD_KH_KW = OD*OH*OW*IC*KD*KH*KW; + pc.OH_OW_IC_KD_KH_KW = OH*OW*IC*KD*KH*KW; + pc.OW_IC_KD_KH_KW = OW*IC*KD*KH*KW; - ggml_pipeline_allocate_descriptor_sets(ctx); + ggml_vk_op_f32<vk_op_im2col_3d_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_IM2COL_3D, std::move(pc)); +} - ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz); +void ggml_vk_timestep_embedding(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + const uint32_t dim = dst->op_params[0]; + const uint32_t max_period = dst->op_params[1]; + const uint32_t nb1 = dst->nb[1] / ggml_type_size(dst->type); - vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool); - ggml_vk_ctx_begin(ctx->device, subctx); - const std::vector<uint32_t> pc = { 1, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne }; - ggml_vk_dispatch_pipeline(ctx, subctx, p, { vk_subbuffer{ qx_buf, 0, qx_sz }, vk_subbuffer{ x_buf, 0, x_sz_f16 } }, pc, { (uint32_t)ne, 1, 1}); - ggml_vk_ctx_end(subctx); + ggml_vk_op_f32<vk_op_timestep_embedding_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_TIMESTEP_EMBEDDING, { + nb1, dim, max_period, + }); +} - auto begin = std::chrono::high_resolution_clock::now(); +void ggml_vk_conv_transpose_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + // src0: (K, Cout, Cin, 1) -- kernel + // src1: (L, Cin, 1, 1) -- input + // dst: (*, Cout, 1, 1) - ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device); - ctx->device->device.resetFences({ ctx->fence }); - ggml_vk_queue_command_pools_cleanup(ctx->device); + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); - auto end = std::chrono::high_resolution_clock::now(); + GGML_TENSOR_BINARY_OP_LOCALS - double ms_dequant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0; - ggml_vk_buffer_read(x_buf, 0, x_chk, x_sz_f16); + GGML_ASSERT(nb00 == sizeof(float)); + GGML_ASSERT(nb10 == sizeof(float)); - int first_err = -1; + const int32_t s0 = dst->op_params[0]; - double avg_err = 0.0; - for (size_t i = 0; i < ne; i++) { - double error = std::fabs(x_ref[i] - ggml_fp16_to_fp32(x_chk[i])); - avg_err += error; + vk_op_conv_transpose_1d_push_constants p{}; + p.Cout = static_cast<uint32_t>(ne01); + p.Cin = static_cast<uint32_t>(ne02); + p.K = static_cast<uint32_t>(ne00); + p.L = static_cast<uint32_t>(ne10); + p.KL = static_cast<uint32_t>(ne0); + p.nb01 = static_cast<uint32_t>(nb01 / nb00); + p.nb02 = static_cast<uint32_t>(nb02 / nb00); + p.nb11 = static_cast<uint32_t>(nb11 / nb10); + p.nb1 = static_cast<uint32_t>(nb1 / nb0); + p.s0 = static_cast<uint32_t>(s0); - if (first_err < 0 && error > 0.05) { - first_err = i; - } - } + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_TRANSPOSE_1D, std::move(p)); +} - avg_err /= ne; +void ggml_vk_col2im_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + // src0: [K_OC, T_in] columns from matmul + // dst: [T_out, OC] - std::cerr << "TEST DEQUANT " << ggml_type_name(quant) << " time=" << ms_dequant << "ms avg_err=" << avg_err << std::endl; + const int32_t stride = dst->op_params[0]; + const int32_t oc = dst->op_params[1]; + const int32_t p0 = dst->op_params[2]; - if (avg_err > 0.1) { - std::cerr << "first_error = " << first_err << std::endl; - std::cerr << "Actual result: " << std::endl << std::endl; - for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) { - std::cerr << ggml_fp16_to_fp32(x_chk[i]) << ", "; - } - std::cerr << std::endl << "Expected result: " << std::endl << std::endl; - for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) { - std::cerr << x_ref[i] << ", "; - } - std::cerr << std::endl; - } + const uint32_t K_OC = static_cast<uint32_t>(src0->ne[0]); + const uint32_t T_in = static_cast<uint32_t>(src0->ne[1]); + const uint32_t T_out = static_cast<uint32_t>(dst->ne[0]); + const uint32_t OC = static_cast<uint32_t>(oc); + const uint32_t K = K_OC / OC; - ggml_vk_destroy_buffer(x_buf); - ggml_vk_destroy_buffer(qx_buf); + vk_op_col2im_1d_push_constants p{}; + p.T_out = T_out; + p.OC = OC; + p.K_OC = K_OC; + p.T_in = T_in; + p.K = K; + p.stride = stride; + p.p0 = p0; - free(x); - free(qx); - free(x_ref); - free(x_chk); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_COL2IM_1D, std::move(p)); } -// This does not work without ggml q8_1 quantization support -// -// typedef uint16_t ggml_half; -// typedef uint32_t ggml_half2; -// -// #define QK8_1 32 -// typedef struct { -// union { -// struct { -// ggml_half d; // delta -// ggml_half s; // d * sum(qs[i]) -// } GGML_COMMON_AGGR_S; -// ggml_half2 ds; -// } GGML_COMMON_AGGR_U; -// int8_t qs[QK8_1]; // quants -// } block_q8_1; -// -// static void ggml_vk_test_quantize(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) { -// VK_LOG_DEBUG("ggml_vk_test_quantize(" << ne << ")"); -// GGML_ASSERT(quant == GGML_TYPE_Q8_1); -// -// const size_t x_sz = sizeof(float) * ne; -// const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant); -// float * x = (float *) malloc(x_sz); -// block_q8_1 * qx = (block_q8_1 *)malloc(qx_sz); -// block_q8_1 * qx_res = (block_q8_1 *)malloc(qx_sz); -// vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); -// vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); -// -// for (size_t i = 0; i < ne; i++) { -// x[i] = rand() / (float)RAND_MAX; -// } -// -// vk_pipeline p = ggml_vk_get_quantize_pipeline(ctx, quant); -// -// ggml_pipeline_request_descriptor_sets(ctx, p, 1); -// -// ggml_pipeline_allocate_descriptor_sets(ctx); -// -// ggml_vk_buffer_write(x_buf, 0, x, x_sz); -// -// vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool); -// ggml_vk_ctx_begin(ctx->device, subctx); -// ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, x_buf), ggml_vk_subbuffer(ctx, qx_buf), ne); -// ggml_vk_ctx_end(subctx); -// -// auto begin = std::chrono::high_resolution_clock::now(); -// -// ggml_vk_submit(subctx, ctx->fence); -// VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_quantize waitForFences"); -// ctx->device->device.resetFences({ ctx->fence }); -// ggml_vk_queue_command_pools_cleanup(ctx->device); -// -// auto end = std::chrono::high_resolution_clock::now(); -// -// double ms_quant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0; -// ggml_vk_buffer_read(qx_buf, 0, qx, qx_sz); -// -// ggml_vk_quantize_data(x, qx_res, ne, quant); -// -// int first_err = -1; -// -// for (size_t i = 0; i < ne / 32; i++) { -// double error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d)); -// -// if (first_err < 0 && error > 0.1) { -// first_err = i; -// } -// -// error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s)); -// -// if (first_err < 0 && error > 0.1) { -// first_err = i; -// } -// -// for (size_t j = 0; j < 32; j++) { -// uint64_t error = std::abs(qx_res[i].qs[j] - qx[i].qs[j]); -// -// if (first_err < 0 && error > 1) { -// first_err = i; -// } -// } -// } -// -// std::cerr << "TEST QUANTIZE " << ggml_type_name(quant) << " time=" << ms_quant << "ms " << (first_err == -1 ? "CORRECT" : "INCORRECT") << std::endl; -// -// if (first_err != -1) { -// std::cerr << "first_error = " << first_err << std::endl; -// std::cerr << "Actual result: " << std::endl << std::endl; -// std::cout << "d=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " "; -// for (size_t j = 0; j < 32; j++) { -// std::cout << " qs" << j << "=" << (uint32_t)qx[first_err].qs[j] << " "; -// } -// std::cerr << std::endl << std::endl << "Expected result: " << std::endl << std::endl; -// std::cout << "d=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " "; -// for (size_t j = 0; j < 32; j++) { -// std::cout << " qs" << j << "=" << (uint32_t)qx_res[first_err].qs[j] << " "; -// } -// std::cerr << std::endl; -// } -// -// ggml_vk_destroy_buffer(x_buf); -// ggml_vk_destroy_buffer(qx_buf); -// -// free(x); -// free(qx); -// free(qx_res); -// } +void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) { + const ggml_tensor * mul0 = cgraph->nodes[node_idx + 0]; + const ggml_tensor * sqr = cgraph->nodes[node_idx + 2]; + const ggml_tensor * mul1 = cgraph->nodes[node_idx + 3]; + ggml_tensor * add = cgraph->nodes[node_idx + 4]; -static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, size_t split_k, size_t shader_size, ggml_type quant, bool mmq = false) { - VK_LOG_DEBUG("ggml_vk_test_dequant_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << ggml_type_name(quant) << ")"); - const size_t x_ne = m * k * batch; - const size_t y_ne = k * n * batch; - const size_t d_ne = m * n * batch; + // x carries the full activation shape, a is the broadcast operand + const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; + const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; - vk_matmul_pipeline2 * pipelines; + // mul1 reads sqr and inv_b in either operand order + const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; - if (mmq) { - pipelines = ctx->device->pipeline_dequant_mul_mat_mat_q8_1; - } else { - pipelines = ctx->device->pipeline_dequant_mul_mat_mat; - } - - const bool fp16acc = ctx->device->fp16; - - vk_pipeline p; - std::string shname; - if (shader_size == 0) { - p = fp16acc ? pipelines[quant].f16acc->a_s : pipelines[quant].f32acc->a_s; - shname = std::string(ggml_type_name(quant)) + "_ALIGNED_S"; - } else if (shader_size == 1) { - p = fp16acc ? pipelines[quant].f16acc->a_m : pipelines[quant].f32acc->a_m; - shname = std::string(ggml_type_name(quant)) + "_ALIGNED_M"; - } else if (shader_size == 2) { - p = fp16acc ? pipelines[quant].f16acc->a_l : pipelines[quant].f32acc->a_l; - shname = std::string(ggml_type_name(quant)) + "_ALIGNED_L"; - } else { - GGML_ASSERT(0); + vk_pipeline pipeline = nullptr; + switch (x->type) { + case GGML_TYPE_F32: pipeline = ctx->device->pipeline_snake_f32; break; + case GGML_TYPE_F16: pipeline = ctx->device->pipeline_snake_f16; break; + case GGML_TYPE_BF16: pipeline = ctx->device->pipeline_snake_bf16; break; + default: GGML_ABORT("unsupported type"); } + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, p->align); - - if (mmq || k != kpad) { - if (shader_size == 0) { - p = fp16acc ? pipelines[quant].f16acc->s : pipelines[quant].f32acc->s; - shname = std::string(ggml_type_name(quant)) + "_S"; - } else if (shader_size == 1) { - p = fp16acc ? pipelines[quant].f16acc->m : pipelines[quant].f32acc->m; - shname = std::string(ggml_type_name(quant)) + "_M"; - } else if (shader_size == 2) { - p = fp16acc ? pipelines[quant].f16acc->l : pipelines[quant].f32acc->l; - shname = std::string(ggml_type_name(quant)) + "_L"; - } else { - GGML_ASSERT(0); - } - } + vk_subbuffer x_buf = ggml_vk_tensor_subbuffer(ctx, x); + vk_subbuffer a_buf = ggml_vk_tensor_subbuffer(ctx, a); + vk_subbuffer inv_b_buf = ggml_vk_tensor_subbuffer(ctx, inv_b); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, add); - if (p == nullptr) { - std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl; - return; - } + vk_op_snake_push_constants pc{}; + pc.ne0 = static_cast<uint32_t>(x->ne[0]); + pc.ne1 = static_cast<uint32_t>(x->ne[1]); - const size_t x_sz = sizeof(float) * x_ne; - const size_t y_sz = sizeof(float) * y_ne; - const size_t qx_sz = x_ne * ggml_type_size(quant)/ggml_blck_size(quant); - const size_t qy_sz = mmq ? y_ne * ggml_type_size(GGML_TYPE_Q8_1)/ggml_blck_size(GGML_TYPE_Q8_1) : y_sz; - const size_t d_sz = sizeof(float) * d_ne; - float * x = (float *) malloc(x_sz); - float * y = (float *) malloc(y_sz); - void * qx = malloc(qx_sz); - vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); - vk_buffer y_buf = ggml_vk_create_buffer_check(ctx->device, y_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); - vk_buffer qy_buf = ggml_vk_create_buffer_check(ctx->device, qy_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); - vk_buffer d_buf = ggml_vk_create_buffer_check(ctx->device, d_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal}); - float * d = (float *) malloc(d_sz); - float * d_chk = (float *) malloc(d_sz); + std::array<uint32_t, 3> elements = { pc.ne0, pc.ne1, 1 }; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, a_buf, inv_b_buf, dst_buf }, pc, elements); +} - for (size_t i = 0; i < x_ne; i++) { - x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; - // x[i] = (i % k == i / k) ? 1.0f : 0.0f; - // x[i] = i % k; - } +void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + uint32_t op = static_cast<uint32_t>(dst->op_params[0]); + const int32_t k0 = dst->op_params[1]; + const int32_t s0 = dst->op_params[2]; + const int32_t p0 = dst->op_params[3]; - ggml_vk_quantize_data(x, qx, x_ne, quant); + const uint32_t IL = src0->ne[0]; - for (size_t i = 0; i < y_ne; i++) { - y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; - // y[i] = (i % k == i / k) ? 1.0f : 0.0f; - // y[i] = i % k; - } + const uint32_t N = dst->ne[3] * dst->ne[2]; - if (split_k > 1) { - ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it); + const uint32_t OC = dst->ne[1]; + const uint32_t OL = dst->ne[0]; - if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) { - // Resize buffer - if (ctx->prealloc_split_k != nullptr) { - ggml_vk_destroy_buffer(ctx->prealloc_split_k); - } - ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal}); - } - } - if (mmq) { - vk_pipeline pipeline_quantize_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); - ggml_pipeline_request_descriptor_sets(ctx, pipeline_quantize_q8_1, num_it); - } + const uint32_t parallel_elements = N * OC * OL; - ggml_pipeline_allocate_descriptor_sets(ctx); + ggml_vk_op_f32<vk_op_pool1d_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_POOL_1D, { + IL, OL, OC, + parallel_elements, + op, + k0, s0, p0, + }); +} - ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz); - ggml_vk_buffer_write(y_buf, 0, y, y_sz); +void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + uint32_t op = static_cast<uint32_t>(dst->op_params[0]); + const int32_t k1 = dst->op_params[1]; + const int32_t k0 = dst->op_params[2]; + const int32_t s1 = dst->op_params[3]; + const int32_t s0 = dst->op_params[4]; + const int32_t p1 = dst->op_params[5]; + const int32_t p0 = dst->op_params[6]; - vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool); - ggml_vk_ctx_begin(ctx->device, subctx); - if (mmq) { - for (size_t i = 0; i < num_it; i++) { - ggml_vk_quantize_q8_1(ctx, subctx, { y_buf, 0, y_sz }, { qy_buf, 0, qy_sz }, y_ne); - ggml_vk_matmul( - ctx, subctx, p, { qx_buf, 0, qx_sz }, { qy_buf, 0, qy_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k }, - m, n, k, - k, k, m, k*m, k*n, m*n, - split_k, batch, batch, batch, 1, 1, n - ); - } - } else { - for (size_t i = 0; i < num_it; i++) { - ggml_vk_matmul( - ctx, subctx, p, { qx_buf, 0, qx_sz }, { y_buf, 0, y_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k }, - m, n, k, - k, k, m, k*m, k*n, m*n, - split_k, batch, batch, batch, 1, 1, n - ); - } - } - ggml_vk_ctx_end(subctx); + const uint32_t IH = src0->ne[1]; + const uint32_t IW = src0->ne[0]; - auto begin = std::chrono::high_resolution_clock::now(); + const uint32_t N = dst->ne[3]; - ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device); - ctx->device->device.resetFences({ ctx->fence }); - ggml_vk_queue_command_pools_cleanup(ctx->device); + const uint32_t OC = dst->ne[2]; + const uint32_t OH = dst->ne[1]; + const uint32_t OW = dst->ne[0]; - auto end = std::chrono::high_resolution_clock::now(); + const uint32_t parallel_elements = N * OC * OH * OW; - double time_ms = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0; - ggml_vk_buffer_read(d_buf, 0, d, d_sz); + ggml_vk_op_f32<vk_op_pool2d_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_POOL_2D, { + IW, IH, OW, OH, OC, + parallel_elements, + op, + k0, k1, s0, s1, p0, p1, + }); +} - ggml_init_params iparams = { - /*.mem_size =*/ 1024*1024*1024, - /*.mem_buffer =*/ NULL, - /*.no_alloc =*/ true, - }; +void ggml_vk_conv_2d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); - ggml_context * ggml_ctx = ggml_init(iparams); + GGML_TENSOR_BINARY_OP_LOCALS + GGML_ASSERT(nb00 == sizeof(float) || nb00 == sizeof(ggml_fp16_t)); + GGML_ASSERT(nb10 == sizeof(float)); + GGML_ASSERT(nb0 == sizeof(float)); - ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, quant, k, m, batch); - ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, GGML_TYPE_F32, k, n, batch); - ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml); + bool transpose = dst->op == GGML_OP_CONV_TRANSPOSE_2D; - src0_ggml->data = qx; - src1_ggml->data = y; - tensor_ggml->data = d_chk; + vk_op_conv2d_push_constants p{}; + p.Cout = static_cast<uint32_t>(!transpose ? ne03 : ne02); + p.Cin = static_cast<uint32_t>(!transpose ? ne02 : ne03); + p.N = static_cast<uint32_t>(ne13); + GGML_ASSERT(p.Cout == ne2); + GGML_ASSERT(p.Cin == ne12); - ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx); - ggml_build_forward_expand(cgraph, tensor_ggml); + p.W = static_cast<uint32_t>(ne10); + p.H = static_cast<uint32_t>(ne11); + p.OW = static_cast<uint32_t>(ne0); + p.OH = static_cast<uint32_t>(ne1); - ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1); + p.nb01 = static_cast<uint32_t>(nb01 / nb00); + p.nb02 = static_cast<uint32_t>(nb02 / nb00); + p.nb03 = static_cast<uint32_t>(nb03 / nb00); - ggml_free(ggml_ctx); + p.nb11 = static_cast<uint32_t>(nb11 / nb10); + p.nb12 = static_cast<uint32_t>(nb12 / nb10); + p.nb13 = static_cast<uint32_t>(nb13 / nb10); - double avg_err = 0.0; - int first_err_n = -1; - int first_err_m = -1; - int first_err_b = -1; + p.nb1 = static_cast<uint32_t>(nb1 / nb0); + p.nb2 = static_cast<uint32_t>(nb2 / nb0); + p.nb3 = static_cast<uint32_t>(nb3 / nb0); - for (size_t i = 0; i < m*n*batch; i++) { - double err = std::fabs(d[i] - d_chk[i]); - avg_err += err; + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, dst->op, std::move(p)); +} - if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) { - first_err_b = i / (m * n); - first_err_n = (i % (m * n)) / m; - first_err_m = (i % (m * n)) % m; - } - } +void ggml_vk_conv_3d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); - avg_err /= m * n; + GGML_TENSOR_BINARY_OP_LOCALS + GGML_ASSERT(nb00 == sizeof(float) || nb00 == sizeof(ggml_fp16_t)); + GGML_ASSERT(nb10 == sizeof(float)); + GGML_ASSERT(nb0 == sizeof(float)); - double tflops = 2.0*m*n*k*batch*num_it / (time_ms / 1000.0) / (1000.0*1000.0*1000.0*1000.0); + vk_op_conv3d_push_constants p{}; + p.IC = static_cast<uint32_t>(ggml_get_op_params_i32(dst, 9)); + p.N = static_cast<uint32_t>(ggml_get_op_params_i32(dst, 10)); + p.OC = static_cast<uint32_t>(ggml_get_op_params_i32(dst, 11)); + GGML_ASSERT(src0->ne[3] == (int64_t)p.IC * p.OC); + GGML_ASSERT(src1->ne[3] == (int64_t)p.IC * p.N); + GGML_ASSERT(dst->ne[3] == (int64_t)p.OC * p.N); - std::cerr << "TEST dequant matmul " << shname; - if (mmq) { - std::cerr << " mmq"; - } - std::cerr << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time_ms / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl; + p.IW = static_cast<uint32_t>(ne10); + p.IH = static_cast<uint32_t>(ne11); + p.ID = static_cast<uint32_t>(ne12); + p.OW = static_cast<uint32_t>(ne0); + p.OH = static_cast<uint32_t>(ne1); + p.OD = static_cast<uint32_t>(ne2); - if (avg_err > 0.01 || std::isnan(avg_err)) { - std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl; - std::cerr << "Actual result: " << std::endl << std::endl; - ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); - std::cerr << std::endl; - std::cerr << "Expected result: " << std::endl << std::endl; - ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + // the shader clamps src addresses to p.IC * p.N * p.IW * p.IH * p.ID - 1 in uint32, so the + // total input element count must fit in a uint32. + GGML_ASSERT((uint64_t)p.IC * p.N * p.IW * p.IH * p.ID <= 0xFFFFFFFFull); - std::cerr << "src0: " << std::endl << std::endl; - ggml_vk_print_matrix_area(x, GGML_TYPE_F32, k, m, first_err_m, first_err_n, first_err_b); - std::cerr << std::endl; - std::cerr << "src1: " << std::endl << std::endl; - ggml_vk_print_matrix_area(y, GGML_TYPE_F32, k, n, first_err_m, first_err_n, first_err_b); + p.nb01 = static_cast<uint32_t>(nb01 / nb00); + p.nb02 = static_cast<uint32_t>(nb02 / nb00); + p.nb03 = static_cast<uint32_t>(nb03 / nb00); - if (split_k > 1) { - float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k); - ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k); + p.nb11 = static_cast<uint32_t>(nb11 / nb10); + p.nb12 = static_cast<uint32_t>(nb12 / nb10); + p.nb13 = static_cast<uint32_t>(nb13 / nb10); - std::cerr << "d_buf0: " << std::endl << std::endl; - ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + p.nb1 = static_cast<uint32_t>(nb1 / nb0); + p.nb2 = static_cast<uint32_t>(nb2 / nb0); + p.nb3 = static_cast<uint32_t>(nb3 / nb0); - std::cerr << "d_buf1: " << std::endl << std::endl; - ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_3D, std::move(p)); +} - std::cerr << "d_buf2: " << std::endl << std::endl; - ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); +void ggml_vk_conv_2d_dw(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + vk_op_conv2d_dw_push_constants p{}; + p.ne = ggml_nelements(dst); + p.channels = dst->ne[2]; + p.batches = dst->ne[3]; + p.dst_w = dst->ne[0]; + p.dst_h = dst->ne[1]; + p.src_w = src1->ne[0]; + p.src_h = src1->ne[1]; + p.knl_w = src0->ne[0]; + p.knl_h = src0->ne[1]; + p.stride_x = dst->op_params[0]; + p.stride_y = dst->op_params[1]; + p.pad_x = dst->op_params[2]; + p.pad_y = dst->op_params[3]; + p.dilation_x = dst->op_params[4]; + p.dilation_y = dst->op_params[5]; - std::cerr << "d_buf3: " << std::endl << std::endl; - ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + GGML_ASSERT(src0->ne[3] == p.channels); + GGML_ASSERT(src1->ne[3] == p.batches); - free(split_k_buf); - } - } + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_2D_DW, std::move(p)); +} - ggml_vk_destroy_buffer(qx_buf); - ggml_vk_destroy_buffer(y_buf); - ggml_vk_destroy_buffer(qy_buf); - ggml_vk_destroy_buffer(d_buf); +void ggml_vk_leaky_relu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + const float * op_params = (const float *)dst->op_params; + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); + p.param1 = op_params[0]; - free(x); - free(qx); - free(y); - free(d); - free(d_chk); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LEAKY_RELU, std::move(p)); } -#endif -static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx) { +void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx) { #if defined(GGML_VULKAN_RUN_TESTS) const std::vector<size_t> vals { 512, 512, 128, @@ -18326,11 +14107,7 @@ static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_contex } } -static void ggml_vk_compute_forward(ggml_backend_vk_context* ctx, ggml_cgraph * cgraph, ggml_tensor* tensor, int tensor_idx, bool almost_ready); - -// Returns true if node has enqueued work into the queue, false otherwise -// If submit is true the current all operations queued so far are being submitted to Vulkan to overlap cmdlist creation and GPU execution. -static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int node_idx, ggml_tensor *node_begin, int node_idx_begin, bool last_node, bool almost_ready, bool submit){ +bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int node_idx, ggml_tensor *node_begin, int node_idx_begin, bool last_node, bool almost_ready, bool submit){ ggml_tensor * node = cgraph->nodes[node_idx]; if (ggml_is_empty(node) || ggml_op_is_empty(node->op) || !node->buffer) { return false; @@ -18467,6 +14244,9 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr } } + // closed explicitly below, and by the destructor on the early returns + ggml_vk_debug_label dbg(compute_ctx, cgraph, node_idx, ctx->num_additional_fused_ops); + switch (node->op) { case GGML_OP_REPEAT: ggml_vk_repeat(ctx, compute_ctx, src0, node); @@ -18633,6 +14413,10 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr ggml_vk_topk_moe(ctx, compute_ctx, cgraph, node_idx); break; } + if (ctx->num_additional_fused_ops) { + ggml_vk_unary_mul(ctx, compute_ctx, cgraph, node_idx); + break; + } switch (ggml_get_unary_op(node)) { case GGML_UNARY_OP_ELU: @@ -18727,6 +14511,18 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr case GGML_OP_CUMSUM: ggml_vk_cumsum(ctx, compute_ctx, src0, node); + break; + case GGML_OP_DSV4_HC_COMB: + ggml_vk_dsv4_hc_comb(ctx, compute_ctx, src0, src1, src2, node); + + break; + case GGML_OP_DSV4_HC_PRE: + ggml_vk_dsv4_hc_pre(ctx, compute_ctx, src0, src1, node); + + break; + case GGML_OP_DSV4_HC_POST: + ggml_vk_dsv4_hc_post(ctx, compute_ctx, src0, src1, src2, src3, node); + break; case GGML_OP_MEAN: ggml_vk_mean(ctx, compute_ctx, src0, node); @@ -18868,6 +14664,9 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr return false; } + // the submit path below can end the command buffer, so close the region first + dbg.close(); + ctx->tensor_ctxs[node_idx] = compute_ctx; #if defined(GGML_VULKAN_CHECK_RESULTS) @@ -18935,7 +14734,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr return true; } -static void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, ggml_tensor * tensor, int tensor_idx, bool almost_ready = false) { +void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, ggml_tensor * tensor, int tensor_idx, bool almost_ready) { GGML_UNUSED(cgraph); VK_LOG_DEBUG("ggml_vk_compute_forward(" << tensor << ", name=" << tensor->name << ", op=" << ggml_op_name(tensor->op) << ", type=" << tensor->type << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << ", view_src=" << tensor->view_src << ", view_offs=" << tensor->view_offs << ")"); @@ -19043,8 +14842,7 @@ static void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph * } } -// Clean up after graph processing is done -static void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) { +void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) { VK_LOG_DEBUG("ggml_vk_graph_cleanup()"); ctx->prealloc_y_last_pipeline_used = {}; ctx->prealloc_y_last_tensor_used = nullptr; @@ -19082,8 +14880,7 @@ static void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) { ctx->descriptor_set_idx = 0; } -// Clean up on backend free -static void ggml_vk_cleanup(ggml_backend_vk_context * ctx) { +void ggml_vk_cleanup(ggml_backend_vk_context * ctx) { VK_LOG_DEBUG("ggml_vk_cleanup(" << ctx->name << ")"); // discard any unsubmitted command buffers ctx->compute_ctx.reset(); @@ -19136,13 +14933,13 @@ static void ggml_vk_cleanup(ggml_backend_vk_context * ctx) { } } -static int ggml_vk_get_device_count() { +int ggml_vk_get_device_count() { ggml_vk_instance_init(); return vk_instance.device_indices.size(); } -static void ggml_vk_get_device_description(int device, char * description, size_t description_size) { +void ggml_vk_get_device_description(int device, char * description, size_t description_size) { ggml_vk_instance_init(); std::vector<vk::PhysicalDevice> devices = vk_instance.instance.enumeratePhysicalDevices(); @@ -19153,30 +14950,24 @@ static void ggml_vk_get_device_description(int device, char * description, size_ snprintf(description, description_size, "%s", props.deviceName.data()); } -// backend interface - -#define UNUSED GGML_UNUSED - -// device backend - -static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) { +bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) { return buffer->buft->iface.get_name == ggml_backend_vk_buffer_type_name; } -static void ggml_backend_vk_buffer_free_buffer(ggml_backend_buffer_t buffer) { +void ggml_backend_vk_buffer_free_buffer(ggml_backend_buffer_t buffer) { VK_LOG_MEMORY("ggml_backend_vk_buffer_free_buffer()"); ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; ggml_vk_destroy_buffer(ctx->dev_buffer); delete ctx; } -static void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t buffer) { +void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t buffer) { return vk_ptr_base; UNUSED(buffer); } -static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { +enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { VK_LOG_DEBUG("ggml_backend_vk_buffer_init_tensor(" << buffer << " (" << buffer->context << "), " << tensor << ")"); if (tensor->view_src != nullptr) { GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft); @@ -19184,7 +14975,7 @@ static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t return GGML_STATUS_SUCCESS; } -static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { +void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { VK_LOG_DEBUG("ggml_backend_vk_buffer_memset_tensor(" << buffer << ", " << tensor << ", " << value << ", " << offset << ", " << size << ")"); ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context; vk_buffer buf = buf_ctx->dev_buffer; @@ -19197,7 +14988,7 @@ static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, g ggml_vk_buffer_memset(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, val32, size); } -static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { +void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")"); ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context; vk_buffer buf = buf_ctx->dev_buffer; @@ -19209,7 +15000,7 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size); } -static void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, +void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " << n_copies << ", " << stride_tensor << ", " << stride_data << ")"); @@ -19223,7 +15014,7 @@ static void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, g ggml_vk_buffer_write_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_data, stride_tensor, size, n_copies); } -static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { +void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")"); ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context; @@ -19236,7 +15027,7 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, cons ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size); } -static void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, +void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " << n_copies << ", " << stride_tensor << ", " << stride_data << ")"); @@ -19251,7 +15042,7 @@ static void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, c ggml_vk_buffer_read_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_tensor, stride_data, size, n_copies); } -static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { +bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { if (ggml_nbytes(src) == 0) { return true; } @@ -19272,34 +15063,19 @@ static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, cons UNUSED(buffer); } -static void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { +void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; ggml_vk_buffer_memset(ctx->dev_buffer, 0, value, buffer->size); } -static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = { - /* .free_buffer = */ ggml_backend_vk_buffer_free_buffer, - /* .get_base = */ ggml_backend_vk_buffer_get_base, - /* .init_tensor = */ ggml_backend_vk_buffer_init_tensor, - /* .memset_tensor = */ ggml_backend_vk_buffer_memset_tensor, - /* .set_tensor = */ ggml_backend_vk_buffer_set_tensor, - /* .get_tensor = */ ggml_backend_vk_buffer_get_tensor, - /* .set_tensor_2d = */ ggml_backend_vk_buffer_set_tensor_2d, - /* .get_tensor_2d = */ ggml_backend_vk_buffer_get_tensor_2d, - /* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor, - /* .clear = */ ggml_backend_vk_buffer_clear, - /* .reset = */ NULL, -}; - -// vk buffer type -static const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft) { +const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft) { ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *)buft->context; return ctx->name.c_str(); } -static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { +ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { VK_LOG_MEMORY("ggml_backend_vk_buffer_type_alloc_buffer(" << size << ")"); ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context; @@ -19315,17 +15091,17 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backe return ggml_backend_buffer_init(buft, ggml_backend_vk_buffer_interface, bufctx, size); } -static size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { +size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context; return ctx->device->properties.limits.minStorageBufferOffsetAlignment; } -static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { +size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context; return ctx->device->suballocation_block_size; } -static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { +size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { return ggml_nbytes(tensor); UNUSED(buft); @@ -19341,8 +15117,6 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) { return &dev->buffer_type; } -// host buffer type - static const char * ggml_backend_vk_host_buffer_type_name(ggml_backend_buffer_type_t buft) { return GGML_VK_NAME "_Host"; @@ -19388,8 +15162,6 @@ static size_t ggml_backend_vk_host_buffer_type_get_max_size(ggml_backend_buffer_ UNUSED(buft); } -// Should be changed to return device-specific host buffer type -// but that probably requires changes in llama.cpp ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() { static struct ggml_backend_buffer_type ggml_backend_vk_buffer_type_host = { /* .iface = */ { @@ -19411,16 +15183,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() { return &ggml_backend_vk_buffer_type_host; } - -// backend - static const char * ggml_backend_vk_name(ggml_backend_t backend) { ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; return ctx->name.c_str(); } -static void ggml_backend_vk_free(ggml_backend_t backend) { +void ggml_backend_vk_free(ggml_backend_t backend) { ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")"); @@ -19595,6 +15364,22 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba return false; } + // If the backend is idle, use a CPU copy to avoid GPU synchronization overhead. + static constexpr size_t max_cpu_copy_size = 128 * 1024; + const bool src_backend_synchronous = backend_src->iface.synchronize == nullptr; + const bool transfer_idle = !ctx->device->async_use_transfer_queue || + ctx->transfer_semaphore_last_submitted == ctx->transfer_semaphore.value; + const bool backend_idle = ctx->compute_ctx.expired() && ctx->transfer_ctx.expired() && + !ctx->submit_pending && !ctx->almost_ready_fence_pending && transfer_idle; + const bool dst_host_coherent = + (dst_buf->memory_property_flags & (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent)) == + (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); + + if ((backend_src == backend_dst || src_backend_synchronous) && backend_idle && dst_host_coherent && ggml_nbytes(src) <= max_cpu_copy_size) { + ggml_vk_buffer_write(dst_buf, vk_tensor_offset(dst) + dst->view_offs, src->data, ggml_nbytes(src)); + return true; + } + vk_context cpy_ctx; if (ctx->device->async_use_transfer_queue) { cpy_ctx = ggml_vk_get_transfer_ctx(ctx); @@ -19607,7 +15392,6 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba src->data, ggml_nbytes(src)); } - GGML_UNUSED(backend_src); return false; } @@ -19664,7 +15448,7 @@ static void ggml_vk_node_ts_harvest(ggml_backend_vk_context * ctx) { } } -static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) { +void ggml_vk_synchronize(ggml_backend_vk_context * ctx) { VK_LOG_DEBUG("ggml_vk_synchronize()"); // WP_VK_FENCE_ON_LAST_SUBMIT: ctx->fence was already attached to the graph's @@ -19784,16 +15568,58 @@ void ggml_backend_vk_node_ts_take(ggml_backend_t backend, struct ggml_backend_vk ctx->node_ts = ggml_backend_vk_node_ts{}; } -static bool ggml_vk_is_empty(ggml_tensor * node) { +bool ggml_vk_is_empty(ggml_tensor * node) { return ggml_is_empty(node) || node->op == GGML_OP_NONE || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE; } -static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) { +static bool ggml_vk_can_fuse_unary_mul(const struct ggml_cgraph * cgraph, int unary_idx, int mul_idx) { + const ggml_tensor * unary = cgraph->nodes[unary_idx]; + const ggml_tensor * mul = cgraph->nodes[mul_idx]; + + if (ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)) < 0) { + return false; + } + if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) { + return false; + } + if (unary->type != mul->type) { + return false; + } + if (mul->src[0] != unary && mul->src[1] != unary) { + return false; + } + const ggml_tensor * other = (mul->src[0] == unary) ? mul->src[1] : mul->src[0]; + if (other == nullptr || other->type != unary->type) { + return false; + } + if (!ggml_is_contiguous_1(other) || !ggml_is_contiguous_1(unary->src[0])) { + return false; + } + // fastmod needs src to tile into dst + if (mul->src[0] == unary) { + return ggml_can_repeat(other, unary); + } + return ggml_can_repeat(unary, mul->src[0]); +} + +static bool ggml_vk_can_fuse_unary_mul_pair(const struct ggml_cgraph * cgraph, int node_idx) { + const enum ggml_op ops[] = { GGML_OP_UNARY, GGML_OP_MUL }; + const int outputs[] = { node_idx + 1 }; + return ggml_can_fuse_subgraph(cgraph, node_idx, 2, ops, outputs, 1) && + ggml_vk_can_fuse_unary_mul(cgraph, node_idx, node_idx + 1); +} + +bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) { + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_MUL) { + return ggml_vk_can_fuse_unary_mul_pair(cgraph, node_idx); + } + if (!ggml_can_fuse(cgraph, node_idx, ops)) { return false; } - if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) { + if ((ops.size() == 2 || ops.size() == 3 || ops.size() == 4) && + ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) { // additional constraints specific to this fusion const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; const ggml_tensor *mul = cgraph->nodes[node_idx + 1]; @@ -19815,7 +15641,45 @@ static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct g if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) { return false; } + + if (ops.size() >= 3 && ops.begin()[2] == GGML_OP_ADD) { + const ggml_tensor *add = cgraph->nodes[node_idx + 2]; + const ggml_tensor *residual = add->src[0] == mul ? add->src[1] : add->src[0]; + if (add->src[0] != mul && add->src[1] != mul) { + return false; + } + if (residual->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 || + !ggml_are_same_shape(add, residual) || !ggml_is_contiguous(residual) || + !ggml_is_contiguous(add) || get_misalign_bytes(ctx, residual) != 0) { + return false; + } + + const ggml_tensor *dst = add; + if (ops.size() == 4) { + if (ops.begin()[3] != GGML_OP_MUL) { + return false; + } + + const ggml_tensor *post_mul = cgraph->nodes[node_idx + 3]; + const ggml_tensor *scale = post_mul->src[0] == add ? post_mul->src[1] : post_mul->src[0]; + if (post_mul->src[0] != add && post_mul->src[1] != add) { + return false; + } + // The shader reads data_e[0], so the final multiply must use a scalar. + if (scale->type != GGML_TYPE_F32 || post_mul->type != GGML_TYPE_F32 || + ggml_nelements(scale) != 1 || !ggml_is_contiguous(post_mul) || + get_misalign_bytes(ctx, scale) != 0) { + return false; + } + dst = post_mul; + } + + if (get_misalign_bytes(ctx, dst) != 0) { + return false; + } + } } + auto const &mm_add_ok = [&](const ggml_tensor *mul, const ggml_tensor *add) { const ggml_tensor *bias = add->src[0] == mul ? add->src[1] : add->src[0]; @@ -19941,8 +15805,7 @@ static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct g return true; } -// Match SSM_CONV + UNARY(SILU) or SSM_CONV + ADD + UNARY(SILU). num_extra is 1 or 2. -static bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, +bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, int num_extra) { const ggml_tensor * conv = cgraph->nodes[node_idx]; if (conv->op != GGML_OP_SSM_CONV) { @@ -19997,7 +15860,7 @@ static bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, const return true; } -static bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, +bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, topk_moe_mode mode) { const ggml_tensor * softmax; @@ -20108,7 +15971,6 @@ static bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struc return true; } -// Manual op-sequence match (ggml_can_fuse_subgraph rejects the mask's external reshape/cpy). static bool ggml_vk_match_ops(const struct ggml_cgraph * cgraph, int node_idx, const std::initializer_list<ggml_op> & ops) { if (node_idx + (int) ops.size() > cgraph->n_nodes) { @@ -20125,8 +15987,7 @@ static bool ggml_vk_match_ops(const struct ggml_cgraph * cgraph, int node_idx, return true; } -// True if the qwen4 QSA indexer top-k can be fused at node_idx (the get_rows). -static bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) { +bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) { if (ctx->device->disable_fusion || !ctx->device->pipeline_topk_radix_qsa) { return false; } @@ -20194,14 +16055,13 @@ static bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const struc return true; } -static bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, +bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) { - GGML_UNUSED(ctx); const ggml_tensor *rope = cgraph->nodes[node_idx + 0]; const ggml_tensor *view = cgraph->nodes[node_idx + 1]; const ggml_tensor *set_rows = cgraph->nodes[node_idx + 2]; - // ne3 not tested + // The set_rows epilogue uses one index per ne2 slice and does not encode ne3. if (rope->src[0]->ne[3] != 1) { return false; } @@ -20210,29 +16070,57 @@ static bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, const return false; } - if (set_rows->src[1]->type != GGML_TYPE_I64) { + // The shader reads each aligned I64 index as a uvec2 and uses its low 32 bits. + if (set_rows->src[1]->type != GGML_TYPE_I64 || !ggml_is_contiguous(set_rows->src[1]) || + set_rows->nb[0] != ggml_type_size(set_rows->type) || get_misalign_bytes(ctx, set_rows->src[1]) != 0) { return false; } - // The view should flatten two dims of rope into one dim + // SET_ROWS consumes one flattened [ne0*ne1] row for each ne2 slice. if (!ggml_is_contiguous(view) || - view->ne[0] != rope->ne[0] * rope->ne[1]) { + view->ne[0] != rope->ne[0] * rope->ne[1] || view->ne[1] != rope->ne[2] || + view->ne[2] != 1 || view->ne[3] != 1 || + ggml_nelements(set_rows->src[1]) != rope->ne[2]) { return false; } - // Only norm/neox/mrope shaders have the fusion code + // Only norm/neox/mrope/imrope shaders have the fusion code const int mode = ((const int32_t *) rope->op_params)[2]; - if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX && mode != GGML_ROPE_TYPE_MROPE) { + if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX && + mode != GGML_ROPE_TYPE_MROPE && mode != GGML_ROPE_TYPE_IMROPE) { + return false; + } + + return true; +} + +bool ggml_vk_can_fuse_rms_norm_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, + int node_idx) { + const ggml_tensor * rms = cgraph->nodes[node_idx]; + const ggml_tensor * view = cgraph->nodes[node_idx + 1]; + const ggml_tensor * set_rows = cgraph->nodes[node_idx + 2]; + + // The RMS kernel reads F32 and writes directly to the F32 or F16 SET_ROWS destination. + if (rms->src[0]->type != GGML_TYPE_F32 || rms->type != GGML_TYPE_F32 || + (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16) || + set_rows->src[1]->type != GGML_TYPE_I64 || !ggml_is_contiguous(set_rows->src[1]) || + set_rows->nb[0] != ggml_type_size(set_rows->type) || get_misalign_bytes(ctx, set_rows->src[1]) != 0) { + return false; + } + // As with the ROPE epilogue, each ne2 slice supplies one flattened row and ne3 is not encoded. + if (rms->ne[3] != 1 || !ggml_is_contiguous(rms->src[0]) || !ggml_is_contiguous(view)) { + return false; + } + if (view->ne[0] != rms->ne[0] * rms->ne[1] || view->ne[1] != rms->ne[2] || + view->ne[2] != 1 || view->ne[3] != 1 || + ggml_nelements(set_rows->src[1]) != rms->ne[2]) { return false; } return true; } -// Pattern check for the 5-op Snake fusion: mul -> sin -> sqr -> mul -> add. -// Verifies the chain shape, the closure x_in_add == x_in_mul0, and that -// the broadcast operands a and inv_b share a [1, C] layout. -static bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) { +bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) { GGML_UNUSED(ctx); if (!ggml_can_fuse(cgraph, node_idx, snake_pattern)) { return false; @@ -20288,11 +16176,7 @@ static bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct g return true; } -// Check whether the tensors overlap in memory. -// Fusions can potentially overwrite src tensors in ways that are not prevented -// by ggml-alloc. If the fusion src is being applied in a way that's elementwise -// with the destination, then it's OK for them to overlap if they are exactly equal. -static bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool elementwise) { +bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool elementwise) { ggml_backend_vk_buffer_context * a_buf_ctx = (ggml_backend_vk_buffer_context *)a->buffer->context; vk_buffer a_buf = a_buf_ctx->dev_buffer; ggml_backend_vk_buffer_context * b_buf_ctx = (ggml_backend_vk_buffer_context *)b->buffer->context; @@ -20315,9 +16199,8 @@ static bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b return false; } -static bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, +bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) { - GGML_UNUSED(ctx); const ggml_tensor *rms = cgraph->nodes[node_idx + 0]; const ggml_tensor *mul = cgraph->nodes[node_idx + 1]; const ggml_tensor *rope = cgraph->nodes[node_idx + 2]; @@ -20349,7 +16232,7 @@ static bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, co return true; } -static uint32_t ggml_vk_fuse_multi_add(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) { +uint32_t ggml_vk_fuse_multi_add(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) { const ggml_tensor *first_node = cgraph->nodes[node_idx]; if (first_node->op != GGML_OP_ADD) { @@ -20432,14 +16315,13 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->device->diag_prev_end = -1; if (vk_instance.debug_utils_support) { - vk::DebugUtilsLabelEXT dul = {}; - dul.pLabelName = "ggml_backend_vk_graph_compute"; - dul.color = std::array<float,4>{1.0f, 1.0f, 1.0f, 1.0f}; - - std::lock_guard<vk_queue_handle> guard(*ctx->device->compute_queue->handle); - vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(ctx->device->compute_queue->handle->queue, reinterpret_cast<VkDebugUtilsLabelEXT*>(&dul)); + ctx->device->debug_cmdbuf_idx = 0; } + // queue scope, so it encloses every submit this evaluation makes. + // closed when the function returns + ggml_vk_debug_label queue_dbg(ctx->device->compute_queue->handle.get(), "ggml_backend_vk_graph_compute"); + ctx->prealloc_size_add_rms_partials_offset = 0; ctx->do_add_rms_partials = false; @@ -20654,6 +16536,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->fused_topk_moe_mode = TOPK_MOE_COUNT; ctx->fused_topk_moe_scale = false; ctx->fused_topk_qsa = false; + ctx->fused_rms_norm_mode = RMS_NORM_COUNT; const char *fusion_string {}; if (!ctx->device->disable_fusion) { uint32_t num_adds = ggml_vk_fuse_multi_add(ctx, cgraph, i); @@ -20688,32 +16571,62 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg fusion_string = "MUL_MAT_ID_MUL"; op_srcs_fused_elementwise[0] = false; op_srcs_fused_elementwise[1] = true; - } else if (ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }, { i + 4 }) && + } else if (ggml_can_fuse_subgraph(cgraph, i, rms_norm_mul_rope_view_set_rows_pattern, { i + 4 }) && ggml_check_edges(cgraph, i, rms_norm_mul_rope_view_set_rows_edges) && ggml_vk_can_fuse_rms_norm_mul_rope(ctx, cgraph, i) && ggml_vk_can_fuse_rope_set_rows(ctx, cgraph, i + 2)) { ctx->num_additional_fused_ops = 4; + ctx->fused_rms_norm_mode = RMS_NORM_MUL_ROPE_VIEW_SET_ROWS; fusion_string = "RMS_NORM_MUL_ROPE_VIEW_SET_ROWS"; op_srcs_fused_elementwise[0] = false; op_srcs_fused_elementwise[1] = false; op_srcs_fused_elementwise[2] = false; op_srcs_fused_elementwise[3] = false; op_srcs_fused_elementwise[4] = false; - } else if (ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE })&& + } else if (ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE }) && ggml_vk_can_fuse_rms_norm_mul_rope(ctx, cgraph, i)) { ctx->num_additional_fused_ops = 2; + ctx->fused_rms_norm_mode = RMS_NORM_MUL_ROPE; fusion_string = "RMS_NORM_MUL_ROPE"; // rope is approximately elementwise - whole rows are done by a single workgroup and it's row-wise op_srcs_fused_elementwise[0] = false; op_srcs_fused_elementwise[1] = true; op_srcs_fused_elementwise[2] = true; + } else if (ggml_vk_can_fuse(ctx, cgraph, i, rms_norm_mul_add_mul_pattern)) { + ctx->num_additional_fused_ops = 3; + ctx->fused_rms_norm_mode = RMS_NORM_MUL_ADD_MUL; + fusion_string = "RMS_NORM_MUL_ADD_MUL"; + std::fill_n(op_srcs_fused_elementwise, 4, true); + } else if (ggml_vk_can_fuse(ctx, cgraph, i, rms_norm_mul_add_pattern)) { + ctx->num_additional_fused_ops = 2; + ctx->fused_rms_norm_mode = RMS_NORM_MUL_ADD; + fusion_string = "RMS_NORM_MUL_ADD"; + std::fill_n(op_srcs_fused_elementwise, 3, true); + } else if (ggml_can_fuse_subgraph(cgraph, i, rms_norm_view_set_rows_pattern, { i + 2 }) && + ggml_check_edges(cgraph, i, rms_norm_view_set_rows_edges) && + ggml_vk_can_fuse_rms_norm_set_rows(ctx, cgraph, i)) { + ctx->num_additional_fused_ops = 2; + ctx->fused_rms_norm_mode = RMS_NORM_VIEW_SET_ROWS; + fusion_string = "RMS_NORM_VIEW_SET_ROWS"; + std::fill_n(op_srcs_fused_elementwise, 3, false); } else if (ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { ctx->num_additional_fused_ops = 1; + ctx->fused_rms_norm_mode = RMS_NORM_MUL; fusion_string = "RMS_NORM_MUL"; // rms_norm is not elementwise, but whole rows must be consumed and the scale factor computed before // they are overwritten, and one workgroup per row. So close enough. op_srcs_fused_elementwise[0] = true; op_srcs_fused_elementwise[1] = true; + } else if (ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL })) { + ctx->num_additional_fused_ops = 1; + switch (ggml_get_unary_op(cgraph->nodes[i])) { + case GGML_UNARY_OP_GELU: fusion_string = "GELU_MUL"; break; + case GGML_UNARY_OP_SIGMOID: fusion_string = "SIGMOID_MUL"; break; + case GGML_UNARY_OP_SILU: fusion_string = "SILU_MUL"; break; + default: fusion_string = "SOFTPLUS_MUL"; break; + } + op_srcs_fused_elementwise[0] = true; + op_srcs_fused_elementwise[1] = true; } else if (ggml_vk_can_fuse_ssm_conv(ctx, cgraph, i, 2)) { ctx->num_additional_fused_ops = 2; fusion_string = "SSM_CONV_BIAS_SILU"; @@ -20727,7 +16640,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg fusion_string = "SSM_CONV_SILU"; op_srcs_fused_elementwise[0] = false; op_srcs_fused_elementwise[1] = true; - } else if (ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }, { i + 2 }) && + } else if (ggml_can_fuse_subgraph(cgraph, i, rope_view_set_rows_pattern, { i + 2 }) && ggml_check_edges(cgraph, i, rope_view_set_rows_edges) && ggml_vk_can_fuse_rope_set_rows(ctx, cgraph, i)) { ctx->num_additional_fused_ops = 2; @@ -20822,39 +16735,31 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg bool need_disable = false; - // topk_moe often overwrites the source, but for a given row all the src values are - // loaded before anything is stored. If there's only one row, this is safe, so treat - // this as a special case. - bool is_topk_moe_single_row = ctx->fused_topk_moe_mode != TOPK_MOE_COUNT && - ggml_nrows(cgraph->nodes[i]->src[0]) == 1; - - if (!is_topk_moe_single_row) { - for (int j = 0; j < 2; ++j) { - ggml_tensor *dst = output_nodes[j]; - if (!dst) { - continue; - } - // Loop over all srcs of all nodes in the fusion. If the src overlaps - // the destination and the src is not an intermediate node that's being - // elided, then disable fusion. - for (int k = 0; k <= ctx->num_additional_fused_ops; ++k) { - for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { - ggml_tensor *src = cgraph->nodes[i + k]->src[s]; - if (!src || src->op == GGML_OP_NONE) { - continue; - } - if (ggml_vk_tensors_overlap(src, dst, op_srcs_fused_elementwise[k])) { - bool found = false; - for (int n = 0; n < k; ++n) { - if (cgraph->nodes[i + n] == src) { - found = true; - break; - } - } - if (!found) { - need_disable = true; + for (int j = 0; j < 2; ++j) { + ggml_tensor *dst = output_nodes[j]; + if (!dst) { + continue; + } + // Loop over all srcs of all nodes in the fusion. If the src overlaps + // the destination and the src is not an intermediate node that's being + // elided, then disable fusion. + for (int k = 0; k <= ctx->num_additional_fused_ops; ++k) { + for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { + ggml_tensor *src = cgraph->nodes[i + k]->src[s]; + if (!src || src->op == GGML_OP_NONE) { + continue; + } + if (ggml_vk_tensors_overlap(src, dst, op_srcs_fused_elementwise[k])) { + bool found = false; + for (int n = 0; n < k; ++n) { + if (cgraph->nodes[i + n] == src) { + found = true; + break; } } + if (!found) { + need_disable = true; + } } } } @@ -20865,6 +16770,8 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->fused_topk_moe_mode = TOPK_MOE_COUNT; ctx->fused_topk_moe_scale = false; ctx->fused_topk_qsa = false; + ctx->fused_rms_norm_mode = RMS_NORM_COUNT; + fusion_string = nullptr; } } @@ -21128,10 +17035,8 @@ static enum ggml_status ggml_backend_vk_graph_plan_compute( } } -// Sort the graph for improved parallelism. -static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params) +void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params) { - GGML_UNUSED(params); VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)"); ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; @@ -21175,6 +17080,16 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * std::set<ggml_tensor *> used_node_set; int first_unused = 0; + + // scheduled or zero-compute nodes in [lo, hi) + auto const &empty_or_scheduled_between = [&](int lo, int hi) -> bool { + for (int v = lo; v < hi; ++v) { + if (!used[v] && !is_empty(graph->nodes[v])) { + return false; + } + } + return true; + }; while (first_unused < graph->n_nodes) { std::vector<int> current_set; @@ -21202,24 +17117,55 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * while (first_unused < graph->n_nodes && used[first_unused]) { first_unused++; } - return true; + return true; + } + return false; + }; + + auto const &add_pattern_alloc_deps = [&](const std::initializer_list<ggml_op> &pattern, int last_node) { + // Keep external inputs alive through the fused output. + std::set<ggml_tensor *> seen; + for (size_t j = 0; j < pattern.size(); ++j) { + ggml_tensor * node = graph->nodes[first_unused + j]; + for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { + ggml_tensor * src = node->src[s]; + if (src && seen.insert(src).second) { + params->add_alloc_dep(params->user_data, src, graph->nodes[last_node]); + } + } + seen.insert(node); } - return false; }; - if (keep_pattern(topk_moe_early_softmax_norm)) { + auto const &keep_topk_moe_pattern = [&](const std::initializer_list<ggml_op> &pattern) -> bool { + if (!match_pattern(pattern, first_unused)) { + return false; + } + + int last_node = first_unused + (int) pattern.size() - 1; + // Some TOPK_MOE variants fuse a trailing scale. + if (last_node + 1 < graph->n_nodes && graph->nodes[last_node + 1]->op == GGML_OP_SCALE) { + last_node++; + } + + add_pattern_alloc_deps(pattern, last_node); + + return keep_pattern(pattern); + }; + + if (keep_topk_moe_pattern(topk_moe_early_softmax_norm)) { continue; } - if (keep_pattern(topk_moe_sigmoid_norm_bias)) { + if (keep_topk_moe_pattern(topk_moe_sigmoid_norm_bias)) { continue; } - if (keep_pattern(topk_moe_sqrt_softplus_norm_bias)) { + if (keep_topk_moe_pattern(topk_moe_sqrt_softplus_norm_bias)) { continue; } - if (keep_pattern(topk_moe_early_softmax)) { + if (keep_topk_moe_pattern(topk_moe_early_softmax)) { continue; } - if (keep_pattern(topk_moe_late_softmax)) { + if (keep_topk_moe_pattern(topk_moe_late_softmax)) { continue; } if (keep_pattern(snake_pattern)) { @@ -21229,6 +17175,22 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * continue; } + if (keep_pattern(rms_norm_mul_add_mul_pattern)) { + continue; + } + if (keep_pattern(rms_norm_mul_add_pattern)) { + continue; + } + if (keep_pattern(rms_norm_mul_rope_view_set_rows_pattern)) { + continue; + } + if (keep_pattern(rms_norm_view_set_rows_pattern)) { + continue; + } + if (keep_pattern(rope_view_set_rows_pattern)) { + continue; + } + // First, grab the next unused node. current_set.push_back(first_unused); @@ -21262,14 +17224,20 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * match_pattern(topk_moe_early_softmax, j) || match_pattern(topk_moe_late_softmax, j) || match_pattern(snake_pattern, j) || - in_qsa_pattern(j)) { + in_qsa_pattern(j) || + match_pattern(rms_norm_mul_add_mul_pattern, j) || + match_pattern(rms_norm_mul_add_pattern, j) || + match_pattern(rms_norm_mul_rope_view_set_rows_pattern, j) || + match_pattern(rms_norm_view_set_rows_pattern, j) || + match_pattern(rope_view_set_rows_pattern, j)) { continue; } bool ok = true; for (int c = first_unused; c < j; ++c) { if (!used[c] && is_src_of(graph->nodes[j], graph->nodes[c]) && - !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_RMS_NORM && graph->nodes[j]->op == GGML_OP_MUL) && + !(c == current_set.back() && graph->nodes[c]->op == GGML_OP_RMS_NORM && graph->nodes[j]->op == GGML_OP_MUL && empty_or_scheduled_between(c+1, j)) && + !(c == current_set.back() && graph->nodes[c]->op == GGML_OP_UNARY && graph->nodes[j]->op == GGML_OP_MUL && empty_or_scheduled_between(c+1, j)) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT && graph->nodes[j]->op == GGML_OP_ADD) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_ADD_ID) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_MUL) && @@ -21302,30 +17270,41 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * } } } - // Look for ROPE + VIEW + SET_ROWS and make them consecutive - if (graph->nodes[rope_idx]->op == GGML_OP_ROPE) { + // Look for ROPE/RMS_NORM + VIEW + SET_ROWS and make them consecutive + if (graph->nodes[rope_idx]->op == GGML_OP_ROPE || graph->nodes[rope_idx]->op == GGML_OP_RMS_NORM) { int view_idx = -1; int set_rows_idx = -1; - for (int k = rope_idx+1; k < std::min(rope_idx + 10, graph->n_nodes); ++k) { - if (view_idx == -1 && - graph->nodes[k]->op == GGML_OP_VIEW && - graph->nodes[k]->src[0] == graph->nodes[rope_idx]) { + for (int k = rope_idx + 1; k < std::min(rope_idx + 15, graph->n_nodes); ++k) { + if (used[k]) { + continue; + } + if (view_idx == -1 && graph->nodes[k]->op == GGML_OP_VIEW && graph->nodes[k]->src[0] == graph->nodes[rope_idx]) { view_idx = k; continue; } - if (view_idx != -1 && - set_rows_idx == -1 && - graph->nodes[k]->op == GGML_OP_SET_ROWS && - graph->nodes[k]->src[0] == graph->nodes[view_idx]) { + if (view_idx != -1 && graph->nodes[k]->op == GGML_OP_SET_ROWS && graph->nodes[k]->src[0] == graph->nodes[view_idx]) { set_rows_idx = k; break; } } if (set_rows_idx != -1) { - current_set.push_back(view_idx); - current_set.push_back(set_rows_idx); - used[view_idx] = true; - used[set_rows_idx] = true; + const int node_idxs[] = { rope_idx, view_idx, set_rows_idx }; + const ggml_op ops[] = { graph->nodes[rope_idx]->op, GGML_OP_VIEW, GGML_OP_SET_ROWS }; + bool can_pull = ggml_can_fuse_subgraph_ext(graph, node_idxs, 3, ops, &set_rows_idx, 1); + + for (int c = rope_idx + 1; can_pull && c < set_rows_idx; ++c) { + if (!used[c] && c != view_idx && !is_empty(graph->nodes[c]) && + is_src_of(graph->nodes[set_rows_idx], graph->nodes[c])) { + can_pull = false; + } + } + + if (can_pull) { + current_set.push_back(view_idx); + current_set.push_back(set_rows_idx); + used[view_idx] = true; + used[set_rows_idx] = true; + } } } // Look for MUL_MAT_ID + ADD_ID + MUL @@ -21371,6 +17350,27 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * } } } + // UNARY + MUL: pull the consuming MUL forward + if (j > 0 && + graph->nodes[j]->op == GGML_OP_UNARY) { + for (int k = j + 1; k < std::min(j + 15, graph->n_nodes); ++k) { + ggml_tensor * mul = graph->nodes[k]; + if (mul->op != GGML_OP_MUL || (mul->src[0] != graph->nodes[j] && mul->src[1] != graph->nodes[j])) { + continue; + } + ggml_tensor * other = (mul->src[0] == graph->nodes[j]) ? mul->src[1] : mul->src[0]; + // the other src must either be weights or already processed + if (!(other->op == GGML_OP_NONE || used_node_set.find(other) != used_node_set.end())) { + continue; + } + if (!ggml_vk_can_fuse_unary_mul(graph, j, k)) { + continue; + } + current_set.push_back(k); + used[k] = true; + break; + } + } } } // Second pass grabs view nodes. @@ -21472,7 +17472,6 @@ static void ggml_backend_vk_event_wait(ggml_backend_t backend, ggml_backend_even } } -// TODO: enable async and synchronize static ggml_backend_i ggml_backend_vk_interface = { /* .get_name = */ ggml_backend_vk_name, /* .free = */ ggml_backend_vk_free, @@ -21621,17 +17620,6 @@ static std::string ggml_backend_vk_get_device_pci_id(int device_idx) { return std::string(pci_bus_id); } -////////////////////////// - -struct ggml_backend_vk_device_context { - size_t device; - std::string name; - std::string description; - bool is_integrated_gpu; - std::string pci_bus_id; - int op_offload_min_batch_size; -}; - static const char * ggml_backend_vk_device_get_name(ggml_backend_dev_t dev) { ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context; return ctx->name.c_str(); @@ -21772,6 +17760,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm // If there's not enough shared memory for row_ids and the result tile, fallback to CPU return false; } + if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { + return false; + } } switch (src0_type) { case GGML_TYPE_F32: @@ -21800,6 +17791,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_TQ1_0: case GGML_TYPE_TQ2_0: break; default: @@ -21968,6 +17960,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_TQ1_0: case GGML_TYPE_TQ2_0: case GGML_TYPE_I32: return true; @@ -22211,6 +18204,31 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm } return false; } + case GGML_OP_DSV4_HC_COMB: + case GGML_OP_DSV4_HC_PRE: + case GGML_OP_DSV4_HC_POST: + { + if (op->type != GGML_TYPE_F32) { + return false; + } + for (uint32_t i = 0; i < GGML_MAX_SRC; ++i) { + if (op->src[i] && op->src[i]->type != GGML_TYPE_F32) { + return false; + } + } + // hc is hardcoded to 4 in the shaders. ggml only constrains it + // to 4 for COMB, so PRE/POST have to be checked here. + if (op->op == GGML_OP_DSV4_HC_PRE && op->src[0]->ne[1] != 4) { + return false; + } + if (op->op == GGML_OP_DSV4_HC_POST && op->src[1]->ne[1] != 4) { + return false; + } + if (op->op == GGML_OP_DSV4_HC_COMB) { + return device->pipeline_dsv4_hc_comb_f32 != nullptr; + } + return true; + } case GGML_OP_SOLVE_TRI: { if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32) { @@ -22407,21 +18425,6 @@ static bool ggml_backend_vk_device_supports_buft(ggml_backend_dev_t dev, ggml_ba return buft_ctx->device->idx == ctx->device; } -static int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) { - switch (op->op) { - case GGML_OP_GET_ROWS: - return 0; - case GGML_OP_MUL_MAT: - return op->ne[1]; - case GGML_OP_MUL_MAT_ID: - case GGML_OP_ROPE: - case GGML_OP_ROPE_BACK: - return op->ne[2]; - default: - return ggml_nrows(op); - } -} - static bool ggml_backend_vk_device_offload_op(ggml_backend_dev_t dev, const ggml_tensor * op) { ggml_backend_vk_device_context * dev_ctx = (ggml_backend_vk_device_context *)dev->context; @@ -22503,31 +18506,6 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm } } -static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size) { - if (!device->external_memory_host) { - return {}; - } - - uintptr_t uptr = reinterpret_cast<uintptr_t>(ptr); - if (uptr & (device->min_imported_host_pointer_alignment - 1)) { - return {}; - } - if (size & (device->min_imported_host_pointer_alignment - 1)) { - return {}; - } - - const vk::MemoryPropertyFlags property_flags = vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached; - - vk_buffer buf {}; - try { - buf = ggml_vk_create_buffer(device, size, { property_flags }, ptr); - } catch (vk::SystemError& e) { - GGML_LOG_WARN("ggml_vulkan: Failed ggml_vk_create_buffer (%s)\n", e.what()); - } - - return buf; -} - static ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { VK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")"); GGML_UNUSED(max_tensor_size); @@ -22658,8 +18636,7 @@ ggml_backend_reg_t ggml_backend_vk_reg() { } } -// Extension availability -static bool ggml_vk_instance_layer_settings_available() { +bool ggml_vk_instance_layer_settings_available() { #ifdef GGML_VULKAN_VALIDATE // Check if validation layer provides the extension const std::string layer_name = "VK_LAYER_KHRONOS_validation"; @@ -22677,7 +18654,8 @@ static bool ggml_vk_instance_layer_settings_available() { #endif return false; } -static bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions) { + +bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions) { #ifdef __APPLE__ // Check for portability enumeration extension for MoltenVK support for (const auto& properties : instance_extensions) { @@ -22692,8 +18670,7 @@ static bool ggml_vk_instance_portability_enumeration_ext_available(const std::ve UNUSED(instance_extensions); } -// Extension availability -static bool ggml_vk_instance_debug_utils_ext_available( +bool ggml_vk_instance_debug_utils_ext_available( const std::vector<vk::ExtensionProperties> & instance_extensions) { // Check for portability enumeration extension for MoltenVK support for (const auto & properties : instance_extensions) { @@ -22708,7 +18685,7 @@ static bool ggml_vk_instance_debug_utils_ext_available( UNUSED(instance_extensions); } -static bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev) { +bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev) { VkPhysicalDeviceFeatures2 device_features2; device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; @@ -22722,7 +18699,7 @@ static bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev) { return vk11_features.storageBuffer16BitAccess; } -static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch) { +bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch) { switch (props.vendorID) { case VK_VENDOR_ID_INTEL: // Only allowing Xe2/Xe3 GPU and integrated Xe GPUs at the moment since older hardware (ex. Arc A770) has performance regressions. @@ -22739,7 +18716,7 @@ static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDevicePrope } } -static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev) { +uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev) { VkPhysicalDeviceProperties2 props = vkdev.getProperties2(); if (props.properties.vendorID != VK_VENDOR_ID_INTEL) { @@ -22782,8 +18759,7 @@ static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev) } } -// checks whether lower <= driver_version < upper, with each bound given as xxx.yyyy -static bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor) { +bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor) { #if defined(_WIN32) // Intel Windows encodes xxx.yyyy as [31:14].[13:0]. const uint32_t major = driver_version >> 14; @@ -22803,781 +18779,259 @@ static bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint3 #endif } +GGML_BACKEND_DL_IMPL(ggml_backend_vk_reg) -// checks - -#ifdef GGML_VULKAN_CHECK_RESULTS -static void ggml_vk_print_graph_origin(const ggml_tensor * tensor, std::vector<const ggml_tensor *>& done, int level = 0) { - if (std::find(done.begin(), done.end(), tensor) != done.end() || level > 10) { - return; - } - for (int j = 0; j < level; j++) { - std::cerr << " "; - } - std::cerr << ggml_op_name(tensor->op) << " gpu=" << (tensor->extra != nullptr) << std::endl; - done.push_back(tensor); +// out-of-lined header method definitions - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (tensor->src[i] != nullptr) { - ggml_vk_print_graph_origin(tensor->src[i], done, level + 1); +void vk_queue_handle_synchronized::submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) { + // Workaround for NVIDIA driver bug + std::unique_lock<std::mutex> device_guard; + if (device_submit_mutex) { + device_guard = std::unique_lock<std::mutex>(*device_submit_mutex); + } + std::lock_guard<std::mutex> guard(mutex); + submit_epoch.fetch_add(1, std::memory_order_acq_rel); + try { + queue.submit(submits, fence); + } catch (vk::DeviceLostError &) { + if (auto dev = device.lock()) { + ggml_vk_print_device_lost_info(dev); } + throw; } } -static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, const void * data, int i0, int i1, int i2, int i3) { - if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16 && tensor->type != GGML_TYPE_I32) { - return; +void vk_queue_handle_unsynchronized::submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) { + // Workaround for NVIDIA driver bug + std::unique_lock<std::mutex> device_guard; + if (device_submit_mutex) { + device_guard = std::unique_lock<std::mutex>(*device_submit_mutex); } - i0 = std::max(i0, 5); - i1 = std::max(i1, 5); - i2 = std::max(i2, 0); - i3 = std::max(i3, 0); - fprintf(stderr, " "); - for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { - fprintf(stderr, "%7d ", idx1); - } - fprintf(stderr, "\n"); - for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) { - fprintf(stderr, "%7d: ", idx0); - for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) { - if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) { - float val; - if (tensor->type == GGML_TYPE_F32) { - val = *(const float *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]); - } else if (tensor->type == GGML_TYPE_F16) { - val = ggml_fp16_to_fp32(*(const ggml_fp16_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0])); - } else if (tensor->type == GGML_TYPE_I32) { - val = *(const int32_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]); - } else { - GGML_ABORT("fatal error"); - } - fprintf(stderr, "% 7.2f ", val); - } else { - fprintf(stderr, " "); - } + // Driver guarantees internal synchronization via VK_KHR_internally_synchronized_queues + submit_epoch.fetch_add(1, std::memory_order_acq_rel); + try { + queue.submit(submits, fence); + } catch (vk::DeviceLostError &) { + if (auto dev = device.lock()) { + ggml_vk_print_device_lost_info(dev); } - fprintf(stderr, "\n"); + throw; } } -static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name) { - void * tensor_data = tensor->data; +vk_device_struct::~vk_device_struct() { + VK_LOG_DEBUG("destroy device " << name); - const bool is_gpu = tensor->buffer != nullptr && ggml_backend_buffer_is_vk(tensor->buffer); + device.destroyFence(fence); - if (is_gpu) { - const size_t tensor_size = ggml_nbytes(tensor); - tensor_data = malloc(tensor_size); + ggml_vk_destroy_buffer(sync_staging); + ggml_vk_destroy_buffer(host_read_staging); + ggml_vk_destroy_buffer(wp_fused_batch_scratch); + host_read_source.reset(); - ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context; + if (compute_queue) compute_queue->cmd_pool.destroy(device); + if (transfer_queue) transfer_queue->cmd_pool.destroy(device); - vk_buffer buffer_gpu = buf_ctx->dev_buffer; - ggml_vk_buffer_read(buffer_gpu, vk_tensor_offset(tensor) + tensor->view_offs, tensor_data, tensor_size); - } + // Explicitly clear to ensure queues drop their shared_ptrs to handles + // before the Vulkan logical device instance is destroyed + compute_queue.reset(); + transfer_queue.reset(); - std::cerr << "TENSOR CHECK " << name << " (" << tensor->name << "): " << ggml_op_name(tensor->op) << std::endl; - std::cerr << "tensor=" << tensor << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << std::endl; - if (tensor->src[0] != nullptr) { - std::cerr << "tensor->src[0]=" << tensor->src[0] << " name=" << tensor->src[0]->name << " op=" << ggml_op_name(tensor->src[0]->op) << " type=" << ggml_type_name(tensor->src[0]->type) << " ne0=" << tensor->src[0]->ne[0] << " nb0=" << tensor->src[0]->nb[0] << " ne1=" << tensor->src[0]->ne[1] << " nb1=" << tensor->src[0]->nb[1] << " ne2=" << tensor->src[0]->ne[2] << " nb2=" << tensor->src[0]->nb[2] << " ne3=" << tensor->src[0]->ne[3] << " nb3=" << tensor->src[0]->nb[3] << std::endl; - } - if (tensor->src[1] != nullptr) { - std::cerr << "tensor->src[1]=" << tensor->src[1] << " name=" << tensor->src[1]->name << " op=" << ggml_op_name(tensor->src[1]->op) << " type=" << ggml_type_name(tensor->src[1]->type) << " ne0=" << tensor->src[1]->ne[0] << " nb0=" << tensor->src[1]->nb[0] << " ne1=" << tensor->src[1]->ne[1] << " nb1=" << tensor->src[1]->nb[1] << " ne2=" << tensor->src[1]->ne[2] << " nb2=" << tensor->src[1]->nb[2] << " ne3=" << tensor->src[1]->ne[3] << " nb3=" << tensor->src[1]->nb[3] << std::endl; - } - std::cerr << std::endl << "Result:" << std::endl; - ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0); - std::cerr << std::endl; - std::vector<const ggml_tensor *> done; - ggml_vk_print_graph_origin(tensor, done); + for (auto& pipeline : all_pipelines) { + if (pipeline.expired()) { + continue; + } - if (is_gpu) { - free(tensor_data); + vk_pipeline pl = pipeline.lock(); + ggml_vk_destroy_pipeline(device, pl); } + all_pipelines.clear(); + + device.destroyDescriptorSetLayout(dsl); + + device.destroy(); } -void * comp_result; -size_t comp_size; -size_t comp_nb[GGML_MAX_DIMS]; -size_t check_counter = 0; -static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) { - ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops]; - if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) { +void vk_perf_logger::print_timings(bool force) { + if (timings.empty()) { return; } - - check_counter++; - if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) { + print_count++; + if ((print_count % vk_perf_logger_frequency) != 0 && !force) { return; } + print_count = 0; + uint64_t total_all_op_times = 0; + std::cerr << "----------------\nVulkan Timings:" << std::endl; + for (const auto & t : timings) { + uint64_t total_op_times = 0; + for (const auto & time : t.second) { + total_op_times += time; + } + std::cerr << t.first << ": " << t.second.size() << " x " << (total_op_times / t.second.size() / 1000.0) + << " us = " << (total_op_times / 1000.0) << " us"; + + // If we have as many flops entries as timing entries for the op, then compute and log the flops/S. + auto it = flops.find(t.first); + if (it != flops.end() && (it->second).size() == t.second.size()) { + uint64_t total_op_flops = 0; + for (const auto & elem : it->second) { + total_op_flops += elem; + } + std::cerr << " (" + << (double(total_op_flops) / (1000.0 * 1000.0 * 1000.0)) / + (double(total_op_times) / (1000.0 * 1000.0 * 1000.0)) + << " GFLOPS/s)"; + } - VK_LOG_DEBUG("ggml_vk_check_results_0(" << tensor->name << ")"); - - struct ggml_init_params iparams = { - /*.mem_size =*/ 2ul*1024ul*1024ul*1024ul, - /*.mem_buffer =*/ NULL, - /*.no_alloc =*/ false, - }; - - struct ggml_context * ggml_ctx = ggml_init(iparams); - - std::array<struct ggml_tensor *, GGML_MAX_SRC> src_clone = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; - const char * srci_name[GGML_MAX_SRC] = {"src0", "src1", "src2", "src3", "src4", "src5", "src6", "src7", "src8", "src9"}; - - std::map<ggml_tensor *, ggml_tensor *> cloned_tensors; - std::vector<void *> cloned_mallocs; + total_all_op_times += total_op_times; - struct ggml_tensor * tensor_clone = nullptr; + std::cerr << std::endl; + } - for (int f = 0; f < ctx->num_additional_fused_ops + 1; ++f) { - tensor = cgraph->nodes[tensor_idx + f]; - for (int i = 0; i < GGML_MAX_SRC; i++) { - ggml_tensor * srci = tensor->src[i]; - if (srci == nullptr) { - continue; - } - // If a src tensor has been cloned, use that one - auto it = cloned_tensors.find(srci); - if (it != cloned_tensors.end()) { - src_clone[i] = it->second; - continue; - } - ggml_tensor * srci_clone = ggml_dup_tensor(ggml_ctx, srci); - size_t srci_size = ggml_nbytes(srci); - - src_clone[i] = srci_clone; - void *src_buffer = malloc(srci_size); - cloned_mallocs.push_back(src_buffer); - - srci_clone->data = src_buffer; - if (ggml_backend_buffer_is_host(srci->buffer)) { - memcpy(srci_clone->data, srci->data, srci_size); - memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS); - } else if (ggml_backend_buffer_is_vk(srci->buffer)) { - ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)srci->buffer->context; - vk_buffer& buffer_gpu = buf_ctx->dev_buffer; - uint64_t offset = vk_tensor_offset(srci) + srci->view_offs; - if (!ggml_is_contiguous(srci) && ggml_vk_dim01_contiguous(srci)) { - for (int i3 = 0; i3 < srci->ne[3]; i3++) { - for (int i2 = 0; i2 < srci->ne[2]; i2++) { - const int idx = i3*srci->ne[2] + i2; - ggml_vk_buffer_read(buffer_gpu, offset + idx * srci->nb[2], ((char *)srci_clone->data + idx * srci_clone->nb[2]), srci->ne[1] * srci->nb[1]); - } - } + if (timings.size() > 0) { + std::cerr << "Total time: " << total_all_op_times / 1000.0 << " us." << std::endl; + } - srci_clone->nb[0] = srci->nb[0]; - srci_clone->nb[1] = srci->nb[1]; - for (int i = 2; i < GGML_MAX_DIMS; i++) { - srci_clone->nb[i] = srci_clone->nb[i - 1]*srci_clone->ne[i - 1]; - } - } else { - if (offset + srci_size >= buffer_gpu->size) { - srci_size = buffer_gpu->size - offset; - } - ggml_vk_buffer_read(buffer_gpu, offset, srci_clone->data, srci_size); - memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS); - } - } else { - GGML_ABORT("fatal error"); - } + timings.clear(); + flops.clear(); +} - if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { - ggml_vk_print_tensor(srci, srci_name[i]); - } +std::string vk_perf_logger::get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops) { + *n_flops = ggml_vk_get_node_flops(node); + std::string fusion_str; + if (fusion_name) { + fusion_str = fusion_name + std::string(" "); + } + if (node->op == GGML_OP_UNARY) { + return fusion_str + ggml_unary_op_name(ggml_get_unary_op(node)); + } + if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) { + const uint64_t m = node->ne[0]; + const uint64_t n = node->ne[1]; + const uint64_t k = node->src[1]->ne[0]; + const uint64_t batch = node->ne[2] * node->ne[3]; + std::string name = ggml_op_name(node->op); + // Naming heuristic only; the perf logger has no device handle, so it + // uses the stock cap rather than this device's effective one. + if ((node->op == GGML_OP_MUL_MAT && n <= mul_mat_vec_max_cols_default) || + (node->op == GGML_OP_MUL_MAT_ID && node->src[2]->ne[1] == 1)) { + name += "_VEC"; } - - if (tensor->op == GGML_OP_FLASH_ATTN_EXT) { - const float * params = (const float *)tensor->op_params; - tensor_clone = ggml_flash_attn_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], params[0], params[1], params[2]); - if (src_clone[4]) { - ggml_flash_attn_ext_add_sinks(tensor_clone, src_clone[4]); - } - } else if (tensor->op == GGML_OP_MUL_MAT) { - tensor_clone = ggml_mul_mat(ggml_ctx, src_clone[0], src_clone[1]); - } else if (tensor->op == GGML_OP_MUL_MAT_ID) { - tensor_clone = ggml_mul_mat_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]); - } else if (tensor->op == GGML_OP_SUB) { - tensor_clone = ggml_sub(ggml_ctx, src_clone[0], src_clone[1]); - } else if (tensor->op == GGML_OP_MUL) { - tensor_clone = ggml_mul(ggml_ctx, src_clone[0], src_clone[1]); - } else if (tensor->op == GGML_OP_DIV) { - tensor_clone = ggml_div(ggml_ctx, src_clone[0], src_clone[1]); - } else if (tensor->op == GGML_OP_CONCAT) { - tensor_clone = ggml_concat(ggml_ctx, src_clone[0], src_clone[1], *(int *)tensor->op_params); - } else if (tensor->op == GGML_OP_UPSCALE) { - tensor_clone = ggml_interpolate(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], (ggml_scale_mode) tensor->op_params[0]); - } else if (tensor->op == GGML_OP_SCALE) { - const float * params = (const float *)tensor->op_params; - tensor_clone = ggml_scale_bias(ggml_ctx, src_clone[0], params[0], params[1]); - } else if (tensor->op == GGML_OP_ADD1) { - tensor_clone = ggml_add1(ggml_ctx, src_clone[0], src_clone[1]); - } else if (tensor->op == GGML_OP_ARANGE) { - const float start = ggml_get_op_params_f32(tensor, 0); - const float stop = ggml_get_op_params_f32(tensor, 1); - const float step = ggml_get_op_params_f32(tensor, 2); - tensor_clone = ggml_arange(ggml_ctx, start, stop, step); - } else if (tensor->op == GGML_OP_FILL) { - const float value = ggml_get_op_params_f32(tensor, 0); - tensor_clone = ggml_fill(ggml_ctx, src_clone[0], value); - } else if (tensor->op == GGML_OP_SQR) { - tensor_clone = ggml_sqr(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_SQRT) { - tensor_clone = ggml_sqrt(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_SIN) { - tensor_clone = ggml_sin(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_COS) { - tensor_clone = ggml_cos(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_LOG) { - tensor_clone = ggml_log(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_TRI) { - tensor_clone = ggml_tri(ggml_ctx, src_clone[0], (ggml_tri_type)ggml_get_op_params_i32(tensor, 0)); - } else if (tensor->op == GGML_OP_DIAG) { - tensor_clone = ggml_diag(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_CLAMP) { - const float * params = (const float *)tensor->op_params; - tensor_clone = ggml_clamp(ggml_ctx, src_clone[0], params[0], params[1]); - } else if (tensor->op == GGML_OP_PAD) { - tensor_clone = ggml_pad_ext(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3], - tensor->op_params[4], tensor->op_params[5], tensor->op_params[6], tensor->op_params[7]); - } else if (tensor->op == GGML_OP_PAD_REFLECT_1D) { - tensor_clone = ggml_pad_reflect_1d(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1]); - } else if (tensor->op == GGML_OP_REPEAT) { - tensor_clone = ggml_repeat(ggml_ctx, src_clone[0], tensor); - } else if (tensor->op == GGML_OP_REPEAT_BACK) { - tensor_clone = ggml_repeat_back(ggml_ctx, src_clone[0], tensor); - } else if (tensor->op == GGML_OP_ADD) { - tensor_clone = ggml_add(ggml_ctx, src_clone[0], src_clone[1]); - } else if (tensor->op == GGML_OP_ACC) { - tensor_clone = ggml_acc(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]); - } else if (tensor->op == GGML_OP_SET) { - tensor_clone = ggml_set(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]); - } else if (tensor->op == GGML_OP_NORM) { - tensor_clone = ggml_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params); - } else if (tensor->op == GGML_OP_GROUP_NORM) { - const float * float_params = (const float *)tensor->op_params; - tensor_clone = ggml_group_norm(ggml_ctx, src_clone[0], tensor->op_params[0], float_params[1]); - } else if (tensor->op == GGML_OP_RMS_NORM) { - tensor_clone = ggml_rms_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params); - } else if (tensor->op == GGML_OP_RMS_NORM_BACK) { - const float eps = ((float *) tensor->op_params)[0]; - tensor_clone = ggml_rms_norm_back(ggml_ctx, src_clone[0], src_clone[1], eps); - } else if (tensor->op == GGML_OP_SILU_BACK) { - tensor_clone = ggml_silu_back(ggml_ctx, src_clone[0], src_clone[1]); - } else if (tensor->op == GGML_OP_L2_NORM) { - const float eps = ((float *) tensor->op_params)[0]; - tensor_clone = ggml_l2_norm(ggml_ctx, src_clone[0], eps); - } else if (tensor->op == GGML_OP_SINKHORN_NORM) { - const float eps = ((float *) tensor->op_params)[0]; - const int32_t iters = ggml_get_op_params_i32(tensor, 1); - tensor_clone = ggml_sinkhorn_norm(ggml_ctx, src_clone[0], eps, iters); - } else if (tensor->op == GGML_OP_SOFT_MAX) { - if (tensor->src[1] != nullptr) { - const float * params = (const float *)tensor->op_params; - tensor_clone = ggml_soft_max_ext(ggml_ctx, src_clone[0], src_clone[1], params[0], params[1]); - } else { - tensor_clone = ggml_soft_max(ggml_ctx, src_clone[0]); - } - } else if (tensor->op == GGML_OP_SOFT_MAX_BACK) { - tensor_clone = ggml_soft_max_ext_back(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); - } else if (tensor->op == GGML_OP_DIAG_MASK_INF) { - tensor_clone = ggml_diag_mask_inf(ggml_ctx, src_clone[0], tensor->op_params[0]); - } else if (tensor->op == GGML_OP_ROPE || tensor->op == GGML_OP_ROPE_BACK) { - const int n_dims = ((int32_t *) tensor->op_params)[1]; - const int mode = ((int32_t *) tensor->op_params)[2]; - //const int n_ctx_ggml = ((int32_t *) tensor->op_params)[3]; - const int n_ctx_orig_ggml = ((int32_t *) tensor->op_params)[4]; - const float freq_base = ((float *) tensor->op_params)[5]; - const float freq_scale = ((float *) tensor->op_params)[6]; - const float ext_factor = ((float *) tensor->op_params)[7]; - const float attn_factor = ((float *) tensor->op_params)[8]; - const float beta_fast = ((float *) tensor->op_params)[9]; - const float beta_slow = ((float *) tensor->op_params)[10]; - if (mode & GGML_ROPE_TYPE_MROPE) { - int32_t *sections = ((int32_t *) tensor->op_params) + 11; - if (tensor->op == GGML_OP_ROPE) { - tensor_clone = ggml_rope_multi(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } else { - tensor_clone = ggml_rope_multi_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } - } else { - if (tensor->op == GGML_OP_ROPE) { - tensor_clone = ggml_rope_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } else { - tensor_clone = ggml_rope_ext_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } - } - const int n_offs = ((int32_t *) tensor->op_params)[15]; - if (n_offs != 0) { - tensor_clone = ggml_rope_set_offset(tensor_clone, n_offs); - } - } else if (tensor->op == GGML_OP_UNARY) { - switch (ggml_get_unary_op(tensor)) { - case GGML_UNARY_OP_EXP: - tensor_clone = ggml_exp(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_EXPM1: - tensor_clone = ggml_expm1(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_ELU: - tensor_clone = ggml_elu(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_SILU: - tensor_clone = ggml_silu(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_GELU: - tensor_clone = ggml_gelu(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_GELU_ERF: - tensor_clone = ggml_gelu_erf(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_GELU_QUICK: - tensor_clone = ggml_gelu_quick(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_RELU: - tensor_clone = ggml_relu(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_XIELU: - tensor_clone = ggml_xielu(ggml_ctx, src_clone[0], 0, 0, 0, 0); - ggml_set_op_params_f32(tensor_clone, 1, ggml_get_op_params_f32(tensor, 1)); - ggml_set_op_params_f32(tensor_clone, 2, ggml_get_op_params_f32(tensor, 2)); - ggml_set_op_params_f32(tensor_clone, 3, ggml_get_op_params_f32(tensor, 3)); - ggml_set_op_params_f32(tensor_clone, 4, ggml_get_op_params_f32(tensor, 4)); - break; - case GGML_UNARY_OP_NEG: - tensor_clone = ggml_neg(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_TANH: - tensor_clone = ggml_tanh(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_SIGMOID: - tensor_clone = ggml_sigmoid(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_HARDSIGMOID: - tensor_clone = ggml_hardsigmoid(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_HARDSWISH: - tensor_clone = ggml_hardswish(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_ABS: - tensor_clone = ggml_abs(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_SOFTPLUS: - tensor_clone = ggml_softplus(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_STEP: - tensor_clone = ggml_step(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_ROUND: - tensor_clone = ggml_round(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_CEIL: - tensor_clone = ggml_ceil(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_FLOOR: - tensor_clone = ggml_floor(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_TRUNC: - tensor_clone = ggml_trunc(ggml_ctx, src_clone[0]); - break; - case GGML_UNARY_OP_SGN: - tensor_clone = ggml_sgn(ggml_ctx, src_clone[0]); - break; - default: - std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl; - GGML_ABORT("fatal error"); - } - } else if (tensor->op == GGML_OP_GLU) { - if (src_clone[1] == nullptr) { - tensor_clone = ggml_glu(ggml_ctx, src_clone[0], (ggml_glu_op) tensor->op_params[0], tensor->op_params[1]); - } else { - tensor_clone = ggml_glu_split(ggml_ctx, src_clone[0], src_clone[1], (ggml_glu_op) tensor->op_params[0]); - } - ggml_set_op_params_i32(tensor_clone, 2, ggml_get_op_params_i32(tensor, 2)); - ggml_set_op_params_i32(tensor_clone, 3, ggml_get_op_params_i32(tensor, 3)); - } else if (tensor->op == GGML_OP_CPY || tensor->op == GGML_OP_DUP) { - if (tensor->src[1] == nullptr) { - tensor_clone = ggml_dup(ggml_ctx, src_clone[0]); - tensor_clone->type = tensor->type; - } else { - tensor_clone = ggml_cpy(ggml_ctx, src_clone[0], src_clone[1]); - } - } else if (tensor->op == GGML_OP_CONT) { - tensor_clone = ggml_cont_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); - } else if (tensor->op == GGML_OP_RESHAPE) { - tensor_clone = ggml_reshape_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); - } else if (tensor->op == GGML_OP_VIEW) { - tensor_clone = ggml_view_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], tensor->nb[1], tensor->nb[2], tensor->nb[3], ((int32_t *) tensor->op_params)[0]); - } else if (tensor->op == GGML_OP_PERMUTE) { - int32_t * params = (int32_t *)tensor->op_params; - tensor_clone = ggml_permute(ggml_ctx, src_clone[0], params[0], params[1], params[2], params[3]); - } else if (tensor->op == GGML_OP_TRANSPOSE) { - tensor_clone = ggml_transpose(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_GET_ROWS) { - tensor_clone = ggml_get_rows(ggml_ctx, src_clone[0], src_clone[1]); - } else if (tensor->op == GGML_OP_ARGSORT) { - tensor_clone = ggml_argsort(ggml_ctx, src_clone[0], (ggml_sort_order) *(int *)tensor->op_params); - } else if (tensor->op == GGML_OP_TOP_K) { - tensor_clone = ggml_top_k(ggml_ctx, src_clone[0], tensor->ne[0]); - } else if (tensor->op == GGML_OP_SUM) { - tensor_clone = ggml_sum(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_SUM_ROWS) { - tensor_clone = ggml_sum_rows(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_CUMSUM) { - tensor_clone = ggml_cumsum(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_MEAN) { - tensor_clone = ggml_mean(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_ARGMAX) { - tensor_clone = ggml_argmax(ggml_ctx, src_clone[0]); - } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS) { - tensor_clone = ggml_cross_entropy_loss(ggml_ctx, src_clone[0], src_clone[1]); - } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS_BACK) { - tensor_clone = ggml_cross_entropy_loss_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]); - } else if (tensor->op == GGML_OP_COUNT_EQUAL) { - tensor_clone = ggml_count_equal(ggml_ctx, src_clone[0], src_clone[1]); - } else if (tensor->op == GGML_OP_SOLVE_TRI) { - tensor_clone = ggml_solve_tri(ggml_ctx, src_clone[0], src_clone[1], true, true, false); - } else if (tensor->op == GGML_OP_IM2COL) { - const int32_t s0 = tensor->op_params[0]; - const int32_t s1 = tensor->op_params[1]; - const int32_t p0 = tensor->op_params[2]; - const int32_t p1 = tensor->op_params[3]; - const int32_t d0 = tensor->op_params[4]; - const int32_t d1 = tensor->op_params[5]; - - const bool is_2D = tensor->op_params[6] == 1; - tensor_clone = ggml_im2col(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1, is_2D, tensor->type); - } else if (tensor->op == GGML_OP_IM2COL_3D) { - const int32_t s0 = tensor->op_params[0]; - const int32_t s1 = tensor->op_params[1]; - const int32_t s2 = tensor->op_params[2]; - const int32_t p0 = tensor->op_params[3]; - const int32_t p1 = tensor->op_params[4]; - const int32_t p2 = tensor->op_params[5]; - const int32_t d0 = tensor->op_params[6]; - const int32_t d1 = tensor->op_params[7]; - const int32_t d2 = tensor->op_params[8]; - const int32_t IC = tensor->op_params[9]; - - tensor_clone = ggml_im2col_3d(ggml_ctx, src_clone[0], src_clone[1], IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, tensor->type); - } else if (tensor->op == GGML_OP_TIMESTEP_EMBEDDING) { - const int32_t dim = tensor->op_params[0]; - const int32_t max_period = tensor->op_params[1]; - tensor_clone = ggml_timestep_embedding(ggml_ctx, src_clone[0], dim, max_period); - } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_1D){ - const int32_t s0 = tensor->op_params[0]; - const int32_t p0 = tensor->op_params[1]; - const int32_t d0 = tensor->op_params[2]; - tensor_clone = ggml_conv_transpose_1d(ggml_ctx, src_clone[0], src_clone[1], s0, p0, d0); - } else if (tensor->op == GGML_OP_COL2IM_1D) { - const int32_t stride = tensor->op_params[0]; - const int32_t oc = tensor->op_params[1]; - const int32_t p0 = tensor->op_params[2]; - tensor_clone = ggml_col2im_1d(ggml_ctx, src_clone[0], stride, oc, p0); - } else if (tensor->op == GGML_OP_POOL_1D) { - enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]); - const int32_t k0 = tensor->op_params[1]; - const int32_t s0 = tensor->op_params[2]; - const int32_t p0 = tensor->op_params[3]; - - tensor_clone = ggml_pool_1d(ggml_ctx, src_clone[0], op, k0, s0, p0); - } else if (tensor->op == GGML_OP_POOL_2D) { - enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]); - const int32_t k0 = tensor->op_params[1]; - const int32_t k1 = tensor->op_params[2]; - const int32_t s0 = tensor->op_params[3]; - const int32_t s1 = tensor->op_params[4]; - const int32_t p0 = tensor->op_params[5]; - const int32_t p1 = tensor->op_params[6]; - - tensor_clone = ggml_pool_2d(ggml_ctx, src_clone[0], op, k0, k1, s0, s1, p0, p1); - } else if (tensor->op == GGML_OP_CONV_2D) { - const int32_t s0 = tensor->op_params[0]; - const int32_t s1 = tensor->op_params[1]; - const int32_t p0 = tensor->op_params[2]; - const int32_t p1 = tensor->op_params[3]; - const int32_t d0 = tensor->op_params[4]; - const int32_t d1 = tensor->op_params[5]; - tensor_clone = ggml_conv_2d(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1); - } else if (tensor->op == GGML_OP_CONV_3D) { - const int32_t s0 = tensor->op_params[0]; - const int32_t s1 = tensor->op_params[1]; - const int32_t s2 = tensor->op_params[2]; - const int32_t p0 = tensor->op_params[3]; - const int32_t p1 = tensor->op_params[4]; - const int32_t p2 = tensor->op_params[5]; - const int32_t d0 = tensor->op_params[6]; - const int32_t d1 = tensor->op_params[7]; - const int32_t d2 = tensor->op_params[8]; - const int32_t IC = tensor->op_params[9]; - const int32_t N = tensor->op_params[10]; - const int32_t OC = tensor->op_params[11]; - tensor_clone = ggml_conv_3d_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, s2, p0, p1, p2, d0, d1, d2, IC, N, OC); - } else if (tensor->op == GGML_OP_CONV_2D_DW) { - const int32_t s0 = tensor->op_params[0]; - const int32_t s1 = tensor->op_params[1]; - const int32_t p0 = tensor->op_params[2]; - const int32_t p1 = tensor->op_params[3]; - const int32_t d0 = tensor->op_params[4]; - const int32_t d1 = tensor->op_params[5]; - tensor_clone = ggml_conv_2d_dw_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1); - } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_2D) { - const int32_t s = tensor->op_params[0]; - tensor_clone = ggml_conv_transpose_2d_p0(ggml_ctx, src_clone[0], src_clone[1], s); - } else if (tensor->op == GGML_OP_LEAKY_RELU) { - const float * op_params = (const float *)tensor->op_params; - tensor_clone = ggml_leaky_relu(ggml_ctx, src_clone[0], op_params[0], false); - } else if (tensor->op == GGML_OP_RWKV_WKV6) { - tensor_clone = ggml_rwkv_wkv6(ggml_ctx, src_clone[0], src_clone[1], - src_clone[2], src_clone[3], src_clone[4], src_clone[5]); - } else if (tensor->op == GGML_OP_RWKV_WKV7) { - tensor_clone = ggml_rwkv_wkv7(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], - src_clone[4], src_clone[5], src_clone[6]); - } else if (tensor->op == GGML_OP_GATED_LINEAR_ATTN) { - const float * op_params = (const float *)tensor->op_params; - tensor_clone = ggml_gated_linear_attn(ggml_ctx, src_clone[0], src_clone[1], - src_clone[2], src_clone[3], src_clone[4], op_params[0]); - } else if (tensor->op == GGML_OP_LIGHTNING_INDEXER) { - tensor_clone = ggml_lightning_indexer(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]); - } else if (tensor->op == GGML_OP_GATED_DELTA_NET) { - tensor_clone = ggml_gated_delta_net(ggml_ctx, src_clone[0], src_clone[1], - src_clone[2], src_clone[3], src_clone[4], src_clone[5], - ggml_get_op_params_i32(tensor, 0)); - } else if (tensor->op == GGML_OP_OPT_STEP_ADAMW) { - src_clone[0]->flags = tensor->src[0]->flags; - tensor_clone = ggml_opt_step_adamw(ggml_ctx, src_clone[0], src_clone[1], - src_clone[2], src_clone[3], src_clone[4]); - } else if (tensor->op == GGML_OP_OPT_STEP_SGD) { - src_clone[0]->flags = tensor->src[0]->flags; - tensor_clone = ggml_opt_step_sgd(ggml_ctx, src_clone[0], src_clone[1], - src_clone[2]); - } else if (tensor->op == GGML_OP_ADD_ID) { - tensor_clone = ggml_add_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]); - } else if (tensor->op == GGML_OP_SSM_SCAN) { - const int32_t K = ggml_get_op_params_i32(tensor, 0); - tensor_clone = ggml_ssm_scan(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], - src_clone[3], src_clone[4], src_clone[5], src_clone[6], K); - } else if (tensor->op == GGML_OP_SSM_CONV) { - tensor_clone = ggml_ssm_conv(ggml_ctx, src_clone[0], src_clone[1]); - } else if (tensor->op == GGML_OP_ROLL) { - const int32_t s0 = tensor->op_params[0]; - const int32_t s1 = tensor->op_params[1]; - const int32_t s2 = tensor->op_params[2]; - const int32_t s3 = tensor->op_params[3]; - tensor_clone = ggml_roll(ggml_ctx, src_clone[0], s0, s1, s2, s3); + name += " "; + name += ggml_type_name(node->src[0]->type); + name += " m=" + std::to_string(m) + " n=" + std::to_string(n) + " k=" + std::to_string(k); + if (node->op == GGML_OP_MUL_MAT_ID) { + name += " n_expert=" + std::to_string(node->src[0]->ne[2]); } - else { - std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl; - GGML_ABORT("fatal error"); + if (batch > 1) { + name += " batch=" + std::to_string(batch); } - cloned_tensors[tensor] = tensor_clone; + return fusion_str + name; } - - ggml_cgraph * cgraph_cpu = ggml_new_graph(ggml_ctx); - ggml_build_forward_expand(cgraph_cpu, tensor_clone); - - ggml_graph_compute_with_ctx(ggml_ctx, cgraph_cpu, 8); - - if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { - ggml_vk_print_tensor(tensor_clone, "tensor_clone"); + if (node->op == GGML_OP_CONV_2D || node->op == GGML_OP_CONV_TRANSPOSE_2D) { + std::string name = ggml_op_name(node->op); + const ggml_tensor * knl = node->src[0]; + uint64_t Cout = node->ne[2]; + uint64_t size_K = node->src[1]->ne[2] * knl->ne[0] * knl->ne[1]; + uint64_t size_N = node->ne[3] * node->ne[0] * node->ne[1]; + name += " M=Cout=" + std::to_string(Cout) + ", K=Cin*KW*KH=" + std::to_string(size_K) + + ", N=N*OW*OH=" + std::to_string(size_N); + return fusion_str + name; + } + if (node->op == GGML_OP_RMS_NORM) { + std::string name = ggml_op_name(node->op); + name += "(" + std::to_string(node->ne[0]) + "," + std::to_string(node->ne[1]) + "," + std::to_string(node->ne[2]) + "," + std::to_string(node->ne[3]) + ")"; + return fusion_str + name; } - - comp_size = ggml_nbytes(tensor_clone); - - comp_result = malloc(comp_size); - memcpy(comp_result, tensor_clone->data, comp_size); - memcpy(comp_nb, tensor_clone->nb, sizeof(size_t) * GGML_MAX_DIMS); - - for (auto m : cloned_mallocs) { - free(m); + if (node->op == GGML_OP_FLASH_ATTN_EXT) { + const ggml_tensor * dst = node; + const ggml_tensor * q = node->src[0]; + const ggml_tensor * k = node->src[1]; + const ggml_tensor * v = node->src[2]; + const ggml_tensor * m = node->src[3]; + std::stringstream name; + name << fusion_str; + name << ggml_op_name(node->op) << + " dst(" << dst->ne[0] << "," << dst->ne[1] << "," << dst->ne[2] << "," << dst->ne[3] << "), " << + " q(" << q->ne[0] << "," << q->ne[1] << "," << q->ne[2] << "," << q->ne[3] << "), " << + " k(" << k->ne[0] << "," << k->ne[1] << "," << k->ne[2] << "," << k->ne[3] << "), " << + " v(" << v->ne[0] << "," << v->ne[1] << "," << v->ne[2] << "," << v->ne[3] << "), " << + " m(" << (m?m->ne[0]:0) << "," << (m?m->ne[1]:0) << "," << (m?m->ne[2]:0) << "," << (m?m->ne[3]:0) << ")"; + return name.str(); + } + if (node->op == GGML_OP_TOP_K) { + std::stringstream name; + name << fusion_str; + name << ggml_op_name(node->op) << + " K=" << node->ne[0] << + " (" << node->src[0]->ne[0] << "," << node->src[0]->ne[1] << "," << node->src[0]->ne[2] << "," << node->src[0]->ne[3] << ")"; + return name.str(); + } + return fusion_str + ggml_op_name(node->op); +} + +ggml_backend_vk_buffer_context::~ggml_backend_vk_buffer_context() { + ggml_vk_destroy_buffer(dev_buffer); +} + +ggml_vk_debug_label::ggml_vk_debug_label(vk_context & ctx, const std::string & pipeline_name, uint32_t wg0, uint32_t wg1, uint32_t wg2) { + if (!vk_instance.debug_utils_support || ctx->s == nullptr) { + return; } - - ggml_free(ggml_ctx); - - VK_LOG_DEBUG("END ggml_vk_check_results_0(" << tensor->name << ")"); + begin(ctx, pipeline_name + " (" + std::to_string(wg0) + "," + std::to_string(wg1) + "," + std::to_string(wg2) + ")"); } -static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) { - ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops]; - if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) { +ggml_vk_debug_label::ggml_vk_debug_label(vk_context & ctx, const ggml_cgraph * cgraph, int node_idx, int n_fused) { + if (!vk_instance.debug_utils_support || ctx->s == nullptr) { return; } - - if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) { - return; + std::string name = ggml_op_name(cgraph->nodes[node_idx]->op); + for (int i = 1; i <= n_fused; i++) { + name += "+"; + name += ggml_op_name(cgraph->nodes[node_idx + i]->op); } + name += " "; + name += cgraph->nodes[node_idx]->name; + begin(ctx, name); +} - VK_LOG_DEBUG("ggml_vk_check_results_1(" << tensor->name << ")"); - - ggml_tensor * src0 = tensor->src[0]; - ggml_tensor * src1 = tensor->src[1]; - ggml_tensor * src2 = tensor->src[2]; - ggml_tensor * src3 = tensor->src[3]; - - void * tensor_data = tensor->data; - - if (ggml_backend_buffer_is_vk(tensor->buffer)) { - size_t tensor_size = ggml_nbytes(tensor); - tensor_data = malloc(tensor_size); - - ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context; - - vk_buffer& buffer_gpu = buf_ctx->dev_buffer; - uint64_t offset = vk_tensor_offset(tensor) + tensor->view_offs; - if (offset + tensor_size >= buffer_gpu->size) { - tensor_size = buffer_gpu->size - offset; - } - - ggml_vk_buffer_read(buffer_gpu, offset, tensor_data, tensor_size); - } - - float first_error_result = -1.0f; - float first_error_correct = -1.0f; - std::array<int, 4> first_error = { -1, -1, -1, -1 }; - double avg_err = 0.0; - size_t counter = 0; - - for (int i3 = 0; i3 < tensor->ne[3]; i3++) { - for (int i2 = 0; i2 < tensor->ne[2]; i2++) { - for (int i1 = 0; i1 < tensor->ne[1]; i1++) { - for (int i0 = 0; i0 < tensor->ne[0]; i0++) { - const bool buffer_size_fit = i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0] < comp_size; - float correct = 0.0f; - float result = 0.0f; - - if (buffer_size_fit) { - if (tensor->type == GGML_TYPE_F32) { - correct = *(float *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]); - result = *(float *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]); - } else if (tensor->type == GGML_TYPE_F16) { - correct = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0])); - result = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0])); - } else if (tensor->type == GGML_TYPE_BF16) { - correct = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0])); - result = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0])); - } else if (tensor->type == GGML_TYPE_I32) { - correct = *(int32_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]); - result = *(int32_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]); - } else if (tensor->type == GGML_TYPE_I64) { - correct = *(int64_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]); - result = *(int64_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]); - } else { - std::cerr << "Results check not implemented for type " << ggml_type_name(tensor->type) << std::endl; - } - } else { - std::cerr << "Missing debug code for type " << ggml_type_name(tensor->type) << std::endl; - GGML_ABORT("fatal error"); - } - - if ((std::isnan(correct) != std::isnan(result)) || (std::isinf(correct) != std::isinf(result)) || !buffer_size_fit) { - std::cerr << "ERROR: Invalid value in " << ggml_op_name(tensor->op) << " i3=" << i3 << " i2=" << i2 << " i1=" << i1 << " i0=" << i0 << " result=" << result << " correct=" << correct << " avg_err=" << (avg_err / counter) << std::endl; - std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; - if (src0 != nullptr) { - std::cerr << "src0=" << src0 << " src0->name=" << src0->name << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; - } - if (src1 != nullptr) { - std::cerr << "src1=" << src1 << " src1->name=" << src1->name << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; - } - if (src2 != nullptr) { - std::cerr << "src2=" << src2 << " src2->name=" << src2->name << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl; - } - if (src3 != nullptr) { - std::cerr << "src3=" << src3 << " src3->name=" << src3->name << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl; - } - std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl; - std::cerr << std::endl << "Result:" << std::endl; - ggml_vk_print_tensor_area(tensor, tensor_data, i0, i1, i2, i3); - std::cerr << std::endl << "Correct:" << std::endl; - ggml_vk_print_tensor_area(tensor, comp_result, i0, i1, i2, i3); - std::cerr << std::endl; - std::vector<const ggml_tensor *> done; - ggml_vk_print_graph_origin(tensor, done); - GGML_ABORT("fatal error"); - } - const double denom = std::fabs(correct) > 1.0f ? (std::fabs(correct) > 1e-8 ? std::fabs(correct) : 1e-8) : 1.0f; - if (first_error[0] == -1 && std::fabs(correct - result) / denom > 0.5) { - first_error[0] = i0; - first_error[1] = i1; - first_error[2] = i2; - first_error[3] = i3; - first_error_result = result; - first_error_correct = correct; - } - - // Special case, value is infinite, avoid NaN result in avg_err - // NaN also appears in results, if both are nan error is 0 - if (!std::isinf(correct) && !std::isinf(result) && !std::isnan(correct) && !std::isnan(result)) { - avg_err += std::fabs(correct - result) / denom; - } - counter++; - } - } - } +ggml_vk_debug_label::ggml_vk_debug_label(vk_queue_handle * handle, const char * name) { + if (!vk_instance.debug_utils_support || handle == nullptr) { + return; } + vk::DebugUtilsLabelEXT label = {}; + label.pLabelName = name; + label.color = std::array<float, 4>{1.0f, 1.0f, 1.0f, 1.0f}; - avg_err /= counter; + qhandle = handle; + std::lock_guard<vk_queue_handle> guard(*qhandle); + vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(qhandle->queue, reinterpret_cast<VkDebugUtilsLabelEXT *>(&label)); +} - if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { - std::cerr << "TENSOR CHECK: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl; - std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; - if (src0 != nullptr) { - std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; - } - if (src1 != nullptr) { - std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; - } - if (src2 != nullptr) { - std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl; - } - if (src3 != nullptr) { - std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl; +void ggml_vk_debug_label::close() { + if (subctx != nullptr) { + // close on the current command buffer, which may differ from the one begin used + if (subctx->s != nullptr) { + vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(subctx->s->buffer->buf); } - std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl; - std::cerr << std::endl << "Result:" << std::endl; - ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0); - std::cerr << std::endl << "Correct:" << std::endl; - ggml_vk_print_tensor_area(tensor, comp_result, 5, 5, 0, 0); - std::cerr << std::endl; - std::vector<const ggml_tensor *> done; - ggml_vk_print_graph_origin(tensor, done); + subctx->debug_labels.pop_back(); + subctx = nullptr; } - - if (avg_err > 0.01 || std::isnan(avg_err)) { - std::cerr << "ERROR: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl; - std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; - if (src0 != nullptr) { - std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; - } - if (src1 != nullptr) { - std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; - } - if (src2 != nullptr) { - std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl; - } - if (src3 != nullptr) { - std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl; - } - std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl; - std::cerr << std::endl << "Result:" << std::endl; - ggml_vk_print_tensor_area(tensor, tensor_data, first_error[0], first_error[1], first_error[2], first_error[3]); - std::cerr << std::endl << "Correct:" << std::endl; - ggml_vk_print_tensor_area(tensor, comp_result, first_error[0], first_error[1], first_error[2], first_error[3]); - std::cerr << std::endl; - std::vector<const ggml_tensor *> done; - ggml_vk_print_graph_origin(tensor, done); - GGML_ABORT("fatal error"); - } else { - std::cerr << check_counter << " " << tensor->name << " op=" << ggml_op_name(tensor->op) << " avg_err=" << avg_err << std::endl; + if (qhandle != nullptr) { + std::lock_guard<vk_queue_handle> guard(*qhandle); + vk_instance.pfn_vkQueueEndDebugUtilsLabelEXT(qhandle->queue); + qhandle = nullptr; } +} - free(comp_result); - comp_result = nullptr; - comp_size = 0; - - if (ggml_backend_buffer_is_vk(tensor->buffer)) { - free(tensor_data); +void ggml_vk_debug_label::begin(vk_context & ctx, const std::string & name) { + if (!vk_instance.debug_utils_support || ctx->s == nullptr) { + return; } - - VK_LOG_DEBUG("END ggml_vk_check_results_1(" << tensor->name << ")"); + subctx = ctx.get(); + subctx->debug_labels.push_back(name); + ggml_vk_cmd_label_begin(subctx->s->buffer->buf, subctx->debug_labels.back().c_str()); } -#endif -GGML_BACKEND_DL_IMPL(ggml_backend_vk_reg) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp b/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp index 0fc2b9b72535..4ba63f7aee1f 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp @@ -33,7 +33,11 @@ void argsort(bool needs_bounds_check, const uint row) { const uint row_offset = row * p.ncols; // initialize indices - dst_row[col] = ivec2(col, floatBitsToInt(data_a[row_offset + col])); + ivec2 value = ivec2(col, 0); + if (!needs_bounds_check || col < p.ncols) { + value.y = floatBitsToInt(data_a[row_offset + col]); + } + dst_row[col] = value; barrier(); uint num_outer_loop_iters = NCOLS_PADDED_LOG2; @@ -42,18 +46,20 @@ void argsort(bool needs_bounds_check, const uint row) { [[unroll]] for (uint j = k / 2, inner_idx = 0; inner_idx < num_inner_loop_iters; j /= 2, inner_idx++) { const int ixj = int(col ^ j); - int idx_0 = (col & k) == 0 ? col : ixj; - int idx_1 = (col & k) == 0 ? ixj : col; + if (ixj > col) { + int idx_0 = (col & k) == 0 ? col : ixj; + int idx_1 = (col & k) == 0 ? ixj : col; - ivec2 sh_idx_0 = dst_row[idx_0]; - ivec2 sh_idx_1 = dst_row[idx_1]; - bool idx_0_oob = needs_bounds_check ? sh_idx_0.x >= p.ncols : false; - bool idx_1_oob = needs_bounds_check ? sh_idx_1.x >= p.ncols : false; + ivec2 sh_idx_0 = dst_row[idx_0]; + ivec2 sh_idx_1 = dst_row[idx_1]; + bool idx_0_oob = needs_bounds_check ? sh_idx_0.x >= p.ncols : false; + bool idx_1_oob = needs_bounds_check ? sh_idx_1.x >= p.ncols : false; - if ((idx_0_oob || - (!idx_1_oob && intBitsToFloat(sh_idx_0.y) > intBitsToFloat(sh_idx_1.y))) && (ixj > col)) { - dst_row[idx_0] = sh_idx_1; - dst_row[idx_1] = sh_idx_0; + if (idx_0_oob || + (!idx_1_oob && intBitsToFloat(sh_idx_0.y) > intBitsToFloat(sh_idx_1.y))) { + dst_row[idx_0] = sh_idx_1; + dst_row[idx_1] = sh_idx_0; + } } barrier(); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp b/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp index 920bac6bb899..f6a29be29ac9 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp @@ -27,6 +27,8 @@ layout (push_constant) uniform parameter { uint inner_end; } p; +shared int s; + void argsort(bool needs_bounds_check, const uint row) { // bitonic sort int col = int(gl_GlobalInvocationID.x); @@ -35,6 +37,12 @@ void argsort(bool needs_bounds_check, const uint row) { const uint row_offset = row * p.ncols; uint idx_offset = row * p.ncols_padded; + // workaround for NV driver/compiler bug - dummy use of shared memory + if (gl_LocalInvocationIndex == 0) { + s = 0; + col += s; + } + bool need_barrier = false; // initialize indices @@ -42,7 +50,10 @@ void argsort(bool needs_bounds_check, const uint row) { [[unroll]] for (int u = 0; u < WG_UNROLL_FACTOR; ++u) { uint c = u*BLOCK_SIZE + col; if (c < p.ncols_padded) { - ivec2 v = ivec2(c, floatBitsToInt(data_a[row_offset + c])); + ivec2 v = ivec2(c, 0); + if (!needs_bounds_check || c < p.ncols) { + v.y = floatBitsToInt(data_a[row_offset + c]); + } tmp_idx[idx_offset + c] = v; } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp b/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp index ef659959d950..06a50181c434 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp @@ -30,9 +30,14 @@ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {uint data_a[];}; layout (binding = 1) writeonly buffer D {uint data_d[];}; -shared uint vals[BLOCK_SIZE]; -shared uint offsets[BLOCK_SIZE]; -shared uint cursors[BLOCK_SIZE]; +// Upper bound on n_experts for the hoisted row-id path. Must match the limit in +// ggml_vk_mul_mat_id_q_f16 (hoist_row_ids). The non-hoisted reduction below only +// needs BLOCK_SIZE entries. +#define MAX_EXPERTS 1024 + +shared uint vals[MAX_EXPERTS]; +shared uint offsets[MAX_EXPERTS]; +shared uint cursors[MAX_EXPERTS]; // data_d layout when p.hoist_row_ids is set: // [0, n_experts) per-expert row count @@ -46,8 +51,8 @@ void main() { const uint tid = gl_LocalInvocationID.x; if (p.hoist_row_ids != 0) { - if (tid < p.n_experts) { - vals[tid] = 0; + for (uint e = tid; e < p.n_experts; e += BLOCK_SIZE) { + vals[e] = 0; } barrier(); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl index 627932bd3547..9df66cb44f9d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl @@ -608,6 +608,21 @@ vec2 get_dm(uint ib, uint a_offset) { } #endif +#if defined(DATA_A_TQ1_0) +float tq1_0_val(uint ib, uint e, uint a_offset) { + const uint bidx = tq1_0_byte_of(e); + const uint qbyte = uint(bidx < 48u ? data_a[a_offset + ib].qs[bidx] + : data_a[a_offset + ib].qh[bidx - 48u]); + return float(tq1_0_trit(qbyte, tq1_0_digit_of(e))) - 1.0; +} +vec2 dequantize(uint ib, uint iqs, uint a_offset) { + return vec2(tq1_0_val(ib, iqs, a_offset), tq1_0_val(ib, iqs + 1u, a_offset)); +} +vec2 get_dm(uint ib, uint a_offset) { + return vec2(float(data_a[a_offset + ib].d), 0); +} +#endif + #if defined(DATA_A_TQ2_0) vec2 dequantize(uint ib, uint iqs, uint a_offset) { // elem e -> byte qs[(e/128)*32 + e%32], bits 2*((e%128)/32); w = q - 1 (d applied via get_dm) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl index 46cc69cb26ed..cc6e242a90d5 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl @@ -247,6 +247,33 @@ f16vec4 dequantFuncQ8_0_v(const in decodeBufQ8_0 bl, const in uint blockCoords[2 return f16vec4(vec4(qi) * vec4(float(d))); } +layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ1_0 { + block_tq1_0 block; +}; + +float16_t dequantFuncTQ1_0(const in decodeBufTQ1_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) +{ + const uint e = coordInBlock[1]; + const uint bidx = tq1_0_byte_of(e); + const uint qbyte = uint(bidx < 48u ? bl.block.qs[bidx] : bl.block.qh[bidx - 48u]); + const uint xi = tq1_0_trit(qbyte, tq1_0_digit_of(e)); + return bl.block.d * (float16_t(int(xi)) - float16_t(1.0)); +} + +f16vec4 dequantFuncTQ1_0_v(const in decodeBufTQ1_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) +{ + const uint e = coordInBlock[1]; + f16vec4 v; + [[unroll]] for (uint k = 0u; k < 4u; ++k) { + const uint ee = e + k; + const uint bidx = tq1_0_byte_of(ee); + const uint qbyte = uint(bidx < 48u ? bl.block.qs[bidx] : bl.block.qh[bidx - 48u]); + const uint xi = tq1_0_trit(qbyte, tq1_0_digit_of(ee)); + v[k] = bl.block.d * (float16_t(int(xi)) - float16_t(1.0)); + } + return v; +} + layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ2_0 { block_tq2_0 block; }; @@ -1041,7 +1068,7 @@ float16_t dequantFuncIQ2_S(const in decodeBufIQ2_S bl, const in uint blockCoords const uint scale = (bl.block.scales[ib32] >> ((idx & 0x10) >> 2)) & 0xf; const uint qs = bl.block.qs[ib8]; const uint qh = bl.block.qh[ib32]; - const uint sign = bl.block.qs[QUANT_K / 8 + ib8] >> (idx & 0x6); + const uint sign = bl.block.qs[QUANT_K_IQ2_S / 8 + ib8] >> (idx & 0x6); const float d = float(bl.block.d); const float db = d * 0.25 * (0.5 + scale); @@ -1063,7 +1090,7 @@ f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoords const uint scale = (bl.block.scales[ib32] >> ((idx & 0x10) >> 2)) & 0xf; const uint qs = bl.block.qs[ib8]; const uint qh = bl.block.qh[ib32]; - const uint sb = uint(bl.block.qs[QUANT_K / 8 + ib8]) >> (idx & 0x6u); + const uint sb = uint(bl.block.qs[QUANT_K_IQ2_S / 8 + ib8]) >> (idx & 0x6u); const float d = float(bl.block.d); const float db = d * 0.25 * (0.5 + scale); @@ -1094,7 +1121,7 @@ float16_t dequantFuncIQ3_XXS(const in decodeBufIQ3_XXS bl, const in uint blockCo uint idx = coordInBlock[1]; const uint iqs = (idx & 0xFC) >> 2; // 0..63 - const uint is = QUANT_K / 4 + ((idx & 0xE0) >> 3);// 8 values + const uint is = QUANT_K_IQ3_XXS / 4 + ((idx & 0xE0) >> 3);// 8 values const float d = float(bl.block.d); const uint qs = bl.block.qs[iqs]; @@ -1117,7 +1144,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCo const uint idx = coordInBlock[1]; const uint iqs = idx >> 2; - const uint is = QUANT_K / 4 + ((idx & 0xE0) >> 3); + const uint is = QUANT_K_IQ3_XXS / 4 + ((idx & 0xE0) >> 3); const float d = float(bl.block.d); const uint qs = bl.block.qs[iqs]; @@ -1406,6 +1433,8 @@ f16vec4 dequantFuncNVFP4_v(const in decodeBufNVFP4 bl, const in uint blockCoords #elif defined(DATA_A_Q8_0) #define dequantFuncA dequantFuncQ8_0 #define dequantFuncA_v dequantFuncQ8_0_v +#elif defined(DATA_A_TQ1_0) +#define dequantFuncA dequantFuncTQ1_0 #elif defined(DATA_A_TQ2_0) #define dequantFuncA dequantFuncTQ2_0 #define dequantFuncA_v dequantFuncTQ2_0_v diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq1_0.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq1_0.comp new file mode 100644 index 000000000000..1632e74631d5 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq1_0.comp @@ -0,0 +1,28 @@ +#version 450 + +#include "dequant_head.glsl" + +layout (local_size_x = 256, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer A {block_tq1_0 data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; + +void main() { + const uint i = gl_GlobalInvocationID.x * 4; + + if (i >= p.nel) { + return; + } + + const uint ib = i / QUANT_K_TQ1_0; + const float d = float(data_a[ib].d); + + [[unroll]] for (uint j = 0; j < 4 && (i + j) < p.nel; ++j) { + const uint e = (i + j) % QUANT_K_TQ1_0; + const uint bidx = tq1_0_byte_of(e); + const uint qbyte = uint(bidx < 48u ? data_a[ib].qs[bidx] + : data_a[ib].qh[bidx - 48u]); + const uint xi = tq1_0_trit(qbyte, tq1_0_digit_of(e)); + data_b[i + j] = D_TYPE(d * (float(xi) - 1.0f)); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dsv4_hc_comb.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dsv4_hc_comb.comp new file mode 100644 index 000000000000..f4ac0378a620 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dsv4_hc_comb.comp @@ -0,0 +1,90 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : require +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_shuffle : require + +// 16 lanes per token, indexed idst + hc*isrc: idst in bits 0..1, isrc in bits 2..3, +// so subgroupShuffleXor by 1|2 reduces a row and by 4|8 a column. + +layout(constant_id = 0) const uint SUBGROUP_SIZE = 32; + +layout(local_size_x_id = 0, local_size_y = 4, local_size_z = 1) in; + +layout(push_constant) uniform parameter +{ + uint n_tokens; + + uint nbm0; uint nbm1; // mixes + uint nbs0; // scale + uint nbb0; // base + uint nbd0; uint nbd1; uint nbd2; // dst + + uint m_offset; + uint s_offset; + uint b_offset; + uint d_offset; + + float eps; + uint n_iter; +}; + +layout(binding = 0, std430) readonly buffer M { float data_m[]; }; +layout(binding = 1, std430) readonly buffer S { float data_s[]; }; +layout(binding = 2, std430) readonly buffer B { float data_b[]; }; +layout(binding = 3, std430) writeonly buffer D { float data_d[]; }; + +const uint hc = 4; +const uint comb_offset = 2 * hc; + +const uint TOKENS_PER_SUBGROUP = SUBGROUP_SIZE / 16; + +void main() { + const uint lane = gl_SubgroupInvocationID; + const uint blk = lane >> 4; // which 16-lane block, i.e. which token + const uint idx = lane & 15; // idst + hc*isrc + + const uint sg = gl_WorkGroupID.x * gl_WorkGroupSize.y + gl_SubgroupID; + const uint it = sg * TOKENS_PER_SUBGROUP + blk; + + // no early return, the shuffles need every lane; out-of-range blocks compute a discarded value + const bool in_range = it < n_tokens; + + const float scale_comb = data_s[s_offset + 2 * nbs0]; + + float v = 0.0f; + if (in_range) { + v = data_m[m_offset + (comb_offset + idx) * nbm0 + it * nbm1] * scale_comb + + data_b[b_offset + (comb_offset + idx) * nbb0]; + } + + // Softmax across destinations: the four lanes sharing an isrc. + float vmax = max(v, subgroupShuffleXor(v, 1)); + vmax = max(vmax, subgroupShuffleXor(vmax, 2)); + v = exp(v - vmax); + + float sum = v + subgroupShuffleXor(v, 1); + sum += subgroupShuffleXor(sum, 2); + v = v / sum + eps; + + // Normalize columns: equal destination indices are four lanes apart. + sum = v + subgroupShuffleXor(v, 4); + sum += subgroupShuffleXor(sum, 8); + v /= sum + eps; + + for (uint i = 1; i < n_iter; ++i) { + sum = v + subgroupShuffleXor(v, 1); + sum += subgroupShuffleXor(sum, 2); + v /= sum + eps; + + sum = v + subgroupShuffleXor(v, 4); + sum += subgroupShuffleXor(sum, 8); + v /= sum + eps; + } + + if (in_range) { + const uint idst = idx & 3; + const uint isrc = idx >> 2; + data_d[d_offset + idst * nbd0 + isrc * nbd1 + it * nbd2] = v; + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dsv4_hc_post.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dsv4_hc_post.comp new file mode 100644 index 000000000000..e521fd9d45da --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dsv4_hc_post.comp @@ -0,0 +1,92 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : require + +// Fan one stream back out to hc streams and add the combination-weighted +// residuals: +// +// dst[i0, idst, it] = x[i0, it]*post[idst, it] +// + sum_isrc residual[i0, isrc, it]*comb[idst, isrc, it] +// +// HAS_COMB == 0: identity mixing, each stream keeps its own residual: +// +// dst[i0, idst, it] = x[i0, it]*post[idst, it] + residual[i0, idst, it] + +layout(constant_id = 0) const uint BLOCK_SIZE = 256; +layout(constant_id = 1) const uint HAS_COMB = 1; + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +layout(push_constant) uniform parameter +{ + uint n_embd; + uint n_tokens; + + uint nbx0; uint nbx1; // x + uint nbr0; uint nbr1; uint nbr2; // residual + uint nbp0; uint nbp1; // post + uint nbc0; uint nbc1; uint nbc2; // comb + uint nbd0; uint nbd1; uint nbd2; // dst + + uint x_offset; + uint r_offset; + uint p_offset; + uint c_offset; + uint d_offset; +}; + +layout(binding = 0, std430) readonly buffer X { float data_x[]; }; +layout(binding = 1, std430) readonly buffer R { float data_r[]; }; +layout(binding = 2, std430) readonly buffer P { float data_p[]; }; +layout(binding = 3, std430) readonly buffer C { float data_c[]; }; +layout(binding = 4, std430) writeonly buffer D { float data_d[]; }; + +const uint hc = 4; + +shared float post_s[hc]; +shared float comb_s[hc * hc]; + +void main() { + const uint tid = gl_LocalInvocationID.x; + const uint it = gl_WorkGroupID.y; + + if (tid < hc) { + post_s[tid] = data_p[p_offset + tid * nbp0 + it * nbp1]; + } + if (HAS_COMB == 1 && tid < hc * hc) { + const uint idst = tid & 3; + const uint isrc = tid >> 2; + comb_s[tid] = data_c[c_offset + idst * nbc0 + isrc * nbc1 + it * nbc2]; + } + barrier(); + + // After the barrier, so every invocation reaches it. + const uint i0 = gl_WorkGroupID.x * BLOCK_SIZE + tid; + if (i0 >= n_embd) { + return; + } + + const float xv = data_x[x_offset + i0 * nbx0 + it * nbx1]; + + const uint rb = r_offset + i0 * nbr0 + it * nbr2; + + float r[hc]; + [[unroll]] + for (uint isrc = 0; isrc < hc; ++isrc) { + r[isrc] = data_r[rb + isrc * nbr1]; + } + + [[unroll]] + for (uint idst = 0; idst < hc; ++idst) { + float result = xv * post_s[idst]; + if (HAS_COMB == 1) { + [[unroll]] + for (uint isrc = 0; isrc < hc; ++isrc) { + result = fma(r[isrc], comb_s[idst + hc * isrc], result); + } + } else { + result += r[idst]; + } + data_d[d_offset + i0 * nbd0 + idst * nbd1 + it * nbd2] = result; + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dsv4_hc_pre.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dsv4_hc_pre.comp new file mode 100644 index 000000000000..fa301547de4e --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dsv4_hc_pre.comp @@ -0,0 +1,75 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : require + +// Collapse the hc residual streams of a token into one, weighted per stream: +// +// dst[i0, it] = scale * sum_ih x[i0, ih, it] * weights[ih, it] +// +// GATED: weights is a per-element gate [n_embd, hc, n_tokens], applied as sigmoid: +// +// dst[i0, it] = scale * sum_ih x[i0, ih, it] * sigmoid(gate[i0, ih, it]) + +layout(constant_id = 0) const uint BLOCK_SIZE = 256; +layout(constant_id = 1) const uint GATED = 0; + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +layout(push_constant) uniform parameter +{ + uint n_embd; + uint n_tokens; + + uint nbx0; uint nbx1; uint nbx2; // x + uint nbw0; uint nbw1; uint nbw2; // weights / gate + uint nbd0; uint nbd1; // dst + + uint x_offset; + uint w_offset; + uint d_offset; + + float scale; +}; + +layout(binding = 0, std430) readonly buffer X { float data_x[]; }; +layout(binding = 1, std430) readonly buffer W { float data_w[]; }; +layout(binding = 2, std430) writeonly buffer D { float data_d[]; }; + +const uint hc = 4; + +shared float w[hc]; + +void main() { + const uint tid = gl_LocalInvocationID.x; + const uint it = gl_WorkGroupID.y; + + if (GATED == 0) { + if (tid < hc) { + w[tid] = data_w[w_offset + tid * nbw0 + it * nbw1]; + } + barrier(); + } + + // After the barrier, so every invocation reaches it. + const uint i0 = gl_WorkGroupID.x * BLOCK_SIZE + tid; + if (i0 >= n_embd) { + return; + } + + const uint xb = x_offset + i0 * nbx0 + it * nbx2; + const uint wb = w_offset + i0 * nbw0 + it * nbw2; + + float result = 0.0f; + [[unroll]] + for (uint ih = 0; ih < hc; ++ih) { + float wv; + if (GATED == 1) { + wv = 1.0f / (1.0f + exp(-data_w[wb + ih * nbw1])); + } else { + wv = w[ih]; + } + result = fma(data_x[xb + ih * nbx1], wv, result); + } + + data_d[d_offset + i0 * nbd0 + it * nbd1] = scale * result; +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl index 5192fa5b6726..17b9c72a92dd 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl @@ -1,37 +1,25 @@ #if !defined(GGML_FA_TYPES_COMP) #define GGML_FA_TYPES_COMP -// FaTypeK / FaTypeV spec constant values. These mirror enum ggml_type so the -// host can pass the type directly. Keep in sync with ggml.h. -#define FA_TYPE_F32 0u -#define FA_TYPE_F16 1u -#define FA_TYPE_Q4_0 2u -#define FA_TYPE_Q4_1 3u -#define FA_TYPE_Q5_0 6u -#define FA_TYPE_Q5_1 7u -#define FA_TYPE_Q8_0 8u -#define FA_TYPE_IQ4_NL 20u -#define FA_TYPE_BF16 30u -#define FA_TYPE_Q1_0 41u -#define FA_TYPE_TURBO4_0 43u +#include "ggml_type_ids.glsl" // Number of matrix elements per buffer block, derived from the K/V type spec // constant. F32 is treated as a vec4 "block" of 4 floats. F16 uses block size 1 // and bypasses the dequant path entirely. Quants follow their ggml block sizes. uint fa_block_elems(uint ty) { switch (ty) { - case FA_TYPE_F32: return 4u; - case FA_TYPE_F16: return 1u; - case FA_TYPE_Q4_0: return uint(QUANT_K_Q4_0); - case FA_TYPE_Q4_1: return uint(QUANT_K_Q4_1); - case FA_TYPE_Q5_0: return uint(QUANT_K_Q5_0); - case FA_TYPE_Q5_1: return uint(QUANT_K_Q5_1); - case FA_TYPE_Q8_0: return uint(QUANT_K_Q8_0); - case FA_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL); - case FA_TYPE_BF16: return 1u; - case FA_TYPE_Q1_0: return uint(QUANT_K_Q1_0); // cm2-only, harmless elsewhere - case FA_TYPE_TURBO4_0: return uint(QK_TURBO4); - default: return 1u; + case GGML_TYPE_F32: return 4u; + case GGML_TYPE_F16: return 1u; + case GGML_TYPE_Q4_0: return uint(QUANT_K_Q4_0); + case GGML_TYPE_Q4_1: return uint(QUANT_K_Q4_1); + case GGML_TYPE_Q5_0: return uint(QUANT_K_Q5_0); + case GGML_TYPE_Q5_1: return uint(QUANT_K_Q5_1); + case GGML_TYPE_Q8_0: return uint(QUANT_K_Q8_0); + case GGML_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL); + case GGML_TYPE_BF16: return 1u; + case GGML_TYPE_Q1_0: return uint(QUANT_K_Q1_0); // cm2-only, harmless elsewhere + case GGML_TYPE_TURBO4_0: return uint(QK_TURBO4); + default: return 1u; } } @@ -40,18 +28,18 @@ uint fa_block_elems(uint ty) { // of int32s per 32-element block on the MMQ K path: ints_per_block == 8 / R. uint fa_quant_r_mmq(uint ty) { switch (ty) { - case FA_TYPE_Q4_0: return uint(QUANT_R_Q4_0); - case FA_TYPE_Q4_1: return uint(QUANT_R_Q4_1); - case FA_TYPE_Q5_0: return uint(QUANT_R_Q5_0); - case FA_TYPE_Q5_1: return uint(QUANT_R_Q5_1); - case FA_TYPE_Q8_0: return uint(QUANT_R_Q8_0); + case GGML_TYPE_Q4_0: return uint(QUANT_R_Q4_0); + case GGML_TYPE_Q4_1: return uint(QUANT_R_Q4_1); + case GGML_TYPE_Q5_0: return uint(QUANT_R_Q5_0); + case GGML_TYPE_Q5_1: return uint(QUANT_R_Q5_1); + case GGML_TYPE_Q8_0: return uint(QUANT_R_Q8_0); default: return 1u; } } bool fa_type_needs_shmem(uint ty) { switch (ty) { - case FA_TYPE_IQ4_NL: return true; + case GGML_TYPE_IQ4_NL: return true; default: return false; } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp b/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp index a56be76c61c5..b5cc33322078 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp @@ -8,7 +8,9 @@ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) writeonly buffer D {D_TYPE data_d[];}; void main() { - const uint i = gl_GlobalInvocationID.x; + // 2D grid flattening: each x workgroup covers gl_WorkGroupSize.x elements, + // each y workgroup covers gl_NumWorkGroups.x * gl_WorkGroupSize.x elements. + const uint i = (gl_GlobalInvocationID.y * gl_NumWorkGroups.x * gl_WorkGroupSize.x) + gl_GlobalInvocationID.x; if (i >= p.KX) { return; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp index 0c1b6d0673e9..107d44aaa801 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp @@ -134,7 +134,7 @@ void main() { // Q8_0 K only needs (qd, _); the asymmetric Q4_*/Q5_* family also stores // the row-sum scaled by qd, used in k_dot_correction. - if (FaTypeK == FA_TYPE_Q8_0) { + if (FaTypeK == GGML_TYPE_Q8_0) { if (buf_iqs == 0) { Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0f); } @@ -218,12 +218,14 @@ void main() { uint32_t c = (idx + tid) % Bc; uint32_t r = (idx + tid) / Bc; if (idx + tid < Bc * Br) { - if ((!KV_bounds_check || j * Bc + c < KV) && (!nem1_bounds_check || i * Br + r < p.nem1)) { - FLOAT_TYPE m = FLOAT_TYPE(data_m[m_offset + (i * Br + r) * m_stride + (j * Bc + c)]); + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + c, kcol); + if (kv_active && (!nem1_bounds_check || i * Br + r < p.nem1)) { + FLOAT_TYPE m = FLOAT_TYPE(data_m[m_offset + (i * Br + r) * m_stride + kcol]); masksh[c * masksh_stride + r] = m; max_mask = max(max_mask, float(m)); } else { - masksh[c * masksh_stride + r] = FLOAT_TYPE(0); + masksh[c * masksh_stride + r] = USE_SPARSE ? FLOAT_TYPE(NEG_FLT_MAX_OVER_2) : FLOAT_TYPE(0); } } } @@ -258,14 +260,15 @@ void main() { uint32_t c = (idx + tid) / (HSK / 4); if (idx + gl_WorkGroupSize.x <= Bc * HSK / 4 || c < Bc) { FLOAT_TYPEV4 K_Tf = FLOAT_TYPEV4(0); - if (!KV_bounds_check || j * Bc + c < KV) { + uint32_t kcol; + if (fa_kv_index(j * Bc + c, kcol)) { if (USE_DECODE_K) { - uint coord = (j * Bc + c) * k_stride * BLOCK_SIZE_K + 4 * d; + uint coord = kcol * k_stride * BLOCK_SIZE_K + 4 * d; uint ib = coord / BLOCK_SIZE_K; uint iqs = (coord % BLOCK_SIZE_K); K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K); } else { - K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + c) * k_stride / 4 + d]); + K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + kcol * k_stride / 4 + d]); } } @@ -305,7 +308,9 @@ void main() { } [[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) { - if (KV_bounds_check && j * Bc + c * cols_per_iter + col_tid >= KV) { + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + c * cols_per_iter + col_tid, kcol); + if (!kv_active) { continue; } @@ -313,12 +318,12 @@ void main() { if (SHMEM_STAGING != 0) { K_Tf = kvsh[(c * cols_per_iter + col_tid) * kvsh_stride + (d * D_split + d_tid)]; } else if (USE_DECODE_K) { - uint coord = (j * Bc + c * cols_per_iter + col_tid) * k_stride * BLOCK_SIZE_K + 4 * (d * D_split + d_tid); + uint coord = kcol * k_stride * BLOCK_SIZE_K + 4 * (d * D_split + d_tid); uint ib = coord / BLOCK_SIZE_K; uint iqs = (coord % BLOCK_SIZE_K); K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K); } else { - K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + c * cols_per_iter + col_tid) * k_stride / 4 + d * D_split + d_tid]); + K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + kcol * k_stride / 4 + d * D_split + d_tid]); } [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) { Sf[r][c] = dot_product(Q_cache[r], K_Tf, Sf[r][c]); @@ -327,7 +332,9 @@ void main() { } } else { [[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) { - if (KV_bounds_check && j * Bc + c * cols_per_iter + col_tid >= KV) { + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + c * cols_per_iter + col_tid, kcol); + if (!kv_active) { continue; } @@ -336,12 +343,12 @@ void main() { if (SHMEM_STAGING != 0) { K_Tf = kvsh[(c * cols_per_iter + col_tid) * kvsh_stride + (d * D_split + d_tid)]; } else if (USE_DECODE_K) { - uint coord = (j * Bc + c * cols_per_iter + col_tid) * k_stride * BLOCK_SIZE_K + 4 * (d * D_split + d_tid); + uint coord = kcol * k_stride * BLOCK_SIZE_K + 4 * (d * D_split + d_tid); uint ib = coord / BLOCK_SIZE_K; uint iqs = (coord % BLOCK_SIZE_K); K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K); } else { - K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + c * cols_per_iter + col_tid) * k_stride / 4 + d * D_split + d_tid]); + K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + kcol * k_stride / 4 + d * D_split + d_tid]); } [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) { Sf[r][c] = dot_product(Qf[tile_row(r) * qf_stride + d * D_split + d_tid], K_Tf, Sf[r][c]); @@ -367,7 +374,7 @@ void main() { // Q4_*/Q5_* take the block-8 fast path when one step covers a full // block; Q8_0 always goes through the per-int get_k_qs* helpers // (its qs is byte-packed, not nibble-packed). - const bool block8_fast = (d_per_step == 8) && (FaTypeK != FA_TYPE_Q8_0); + const bool block8_fast = (d_per_step == 8) && (FaTypeK != GGML_TYPE_Q8_0); if (SHMEM_STAGING != 0) { const uint k_block_idx = (d_tid * (HSK_per_thread / 4) + d_block) / 8; @@ -375,7 +382,7 @@ void main() { k_dm = ACC_TYPEV2(kblocksh[buf_ib].dm); if (block8_fast) { - const bool has_qh = (FaTypeK == FA_TYPE_Q5_0) || (FaTypeK == FA_TYPE_Q5_1); + const bool has_qh = (FaTypeK == GGML_TYPE_Q5_0) || (FaTypeK == GGML_TYPE_Q5_1); [[unroll]] for (uint32_t d = 0; d < 4; d++) { uint vui = kblocksh[buf_ib].qs[d]; k_quants[d ] = int32_t( vui & 0x0F0F0F0F); @@ -489,14 +496,15 @@ void main() { uint32_t c = (idx + tid) / (HSV / 4); if (idx + gl_WorkGroupSize.x <= Bc * HSV / 4 || c < Bc) { FLOAT_TYPEV4 V_Tf = FLOAT_TYPEV4(0); - if (!KV_bounds_check || j * Bc + c < KV) { + uint32_t vcol; + if (fa_kv_index(j * Bc + c, vcol)) { if (USE_DECODE_V) { - uint coord = (j * Bc + c) * v_stride * BLOCK_SIZE_V + 4 * d; + uint coord = vcol * v_stride * BLOCK_SIZE_V + 4 * d; uint ib = coord / BLOCK_SIZE_V; uint iqs = (coord % BLOCK_SIZE_V); V_Tf = dequantize4(ib, iqs, v_offset, BINDING_IDX_V); } else { - V_Tf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + (j * Bc + c) * v_stride / 4 + d]); + V_Tf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + vcol * v_stride / 4 + d]); } } @@ -507,7 +515,9 @@ void main() { } [[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) { - if (KV_bounds_check && j * Bc + c * cols_per_iter + col_tid >= KV) { + uint32_t vcol; + bool kv_active = fa_kv_index(j * Bc + c * cols_per_iter + col_tid, vcol); + if (!kv_active) { continue; } @@ -522,12 +532,12 @@ void main() { if (SHMEM_STAGING != 0) { Vf = kvsh[(c * cols_per_iter + col_tid) * kvsh_stride + (d * D_split + d_tid)]; } else if (USE_DECODE_V) { - uint coord = (j * Bc + c * cols_per_iter + col_tid) * v_stride * BLOCK_SIZE_V + 4 * (d * D_split + d_tid); + uint coord = vcol * v_stride * BLOCK_SIZE_V + 4 * (d * D_split + d_tid); uint ib = coord / BLOCK_SIZE_V; uint iqs = (coord % BLOCK_SIZE_V); Vf = dequantize4(ib, iqs, v_offset, BINDING_IDX_V); } else { - Vf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + (j * Bc + c * cols_per_iter + col_tid) * v_stride / 4 + d * D_split + d_tid]); + Vf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + vcol * v_stride / 4 + d * D_split + d_tid]); } [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) { Of[r][d] += FLOAT_TYPEV4(Pf[r] * Vf); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl index 0ce4503a8847..2e0e23bc11c1 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl @@ -24,6 +24,8 @@ const bool USE_MASK_OPT = (Flags & 1) != 0; const bool MASK_ENABLE = (Flags & 2) != 0; const bool LOGIT_SOFTCAP = (Flags & 4) != 0; const bool OLD_AMD_WINDOWS = (Flags & 8) != 0; +// Sparse: gather binding-7 indices instead of scanning [0,KV); p.split_kv = n_kv_max. +const bool USE_SPARSE = (Flags & 16) != 0; // Round up head sizes to a multiple of 16, for coopmat1/coopmat2 paths const uint32_t HSK_pad = (HSK + 15) & ~15; @@ -82,6 +84,8 @@ layout (binding = 5) writeonly buffer OV4 {D_TYPEV4 data_ov4[];}; layout (binding = 6) readonly buffer MO {uint32_t data_mask_opt[];}; +layout (binding = 7) readonly buffer SP {int32_t data_sparse[];}; + #define MASK_OPT_ALL_NEG_INF 1 #define MASK_OPT_ALL_ZERO 2 @@ -105,8 +109,8 @@ layout (binding = 6) readonly buffer MO {uint32_t data_mask_opt[];}; #define BLOCK_SIZE_V fa_block_elems(FaTypeV) // F16 reads f16 elements directly from the binding; everything else routes // through dequantize4 / the MMQ helpers to unpack from the packed block layout. -#define USE_DECODE_K (FaTypeK != FA_TYPE_F16) -#define USE_DECODE_V (FaTypeV != FA_TYPE_F16) +#define USE_DECODE_K (FaTypeK != GGML_TYPE_F16) +#define USE_DECODE_V (FaTypeV != GGML_TYPE_F16) #define CEIL_DIV(a, b) (((a) + (b) - 1) / (b)) @@ -144,7 +148,7 @@ ACC_TYPE perElemOpGetSink(const in uint32_t r, const in uint32_t c, const in ACC uint32_t i, N, KV, split_k_index, Tr, start_j, end_j, gqa_iq1, iq2, iq3, rk2, rk3, rv2, rv3, ik2, ik3, iv2, iv3, - q_stride, k_stride, v_stride, m_stride; + q_stride, k_stride, v_stride, m_stride, sparse_base; void init_indices() { @@ -208,6 +212,33 @@ void init_indices() // that prevents the compiler from folding the "&" through the select // and breaking the alignment detection. m_stride = (p.gqa_ratio > 1) ? (p.gqa_ratio >> 16) : KV; + + // Sparse: the tile shares one mask row (gqa heads, or Br==1). split_k + // partitions the n_kv_max blocks. + if (USE_SPARSE) { + uint32_t qrow = (p.gqa_ratio > 1) ? gqa_iq1 : (i * Br); + sparse_base = (((iq3 % p.nem3) * p.nem2 + (iq2 % p.nem2)) * p.nem1 + qrow) * p.split_kv; + + uint32_t total_blocks = CEIL_DIV(p.split_kv, Bc); + uint32_t per_blocks = CEIL_DIV(total_blocks, p.k_num); + start_j = min(split_k_index * per_blocks, total_blocks); + end_j = min((split_k_index + 1) * per_blocks, total_blocks); + } +} + +// Resolve a linear KV slot to a real column; false for inactive (sparse padding/-1, or dense OOB). +bool fa_kv_index(uint lin, out uint kv_col) { + if (USE_SPARSE) { + if (lin >= p.split_kv) { + kv_col = 0; + return false; + } + int idx = data_sparse[sparse_base + lin]; + kv_col = idx >= 0 ? uint(idx) : 0; + return idx >= 0; + } + kv_col = lin; + return !KV_bounds_check || lin < KV; } // Bias applied to softmax to stay in fp16 range. diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp index 057ed739aa8d..aa9dd624bef3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp @@ -176,9 +176,16 @@ void main() { uint32_t c = (idx + tid) / (Br / 4); uint32_t r = (idx + tid) % (Br / 4); if (idx + tid < Bc * Br / 4 || idx + gl_WorkGroupSize.x <= Bc * Br / 4) { - if ((!KV_bounds_check || j * Bc + c < KV)) { + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + c, kcol); + if (kv_active) { f16vec4 m; - if (!nem1_bounds_check || i * Br + r * 4 + 3 < p.nem1) { + if (USE_SPARSE) { + // sparse is gqa-gated (m_stride == 0): all four rows share the value + FLOAT_TYPE mv = FLOAT_TYPE(data_m[m_offset + kcol]); + m = f16vec4(mv); + max_mask = max(max_mask, float(mv)); + } else if (!nem1_bounds_check || i * Br + r * 4 + 3 < p.nem1) { m = f16vec4(data_m[m_offset + (i * Br + r * 4 ) * m_stride + (j * Bc + c)], data_m[m_offset + (i * Br + r * 4 + 1) * m_stride + (j * Bc + c)], data_m[m_offset + (i * Br + r * 4 + 2) * m_stride + (j * Bc + c)], @@ -206,6 +213,8 @@ void main() { m = f16vec4(0.0); } mask_cache[idx / WorkGroupSize] = m; + } else if (USE_SPARSE) { + mask_cache[idx / WorkGroupSize] = f16vec4(NEG_FLT_MAX_OVER_2); } } } @@ -231,17 +240,19 @@ void main() { uint32_t c = (idx + tid) / (HSK_pad / 4); if (idx + gl_WorkGroupSize.x <= Bc * HSK_pad / 4 || c < Bc) { FLOAT_TYPEV4 K_Tf = FLOAT_TYPEV4(0); - if ((!KV_bounds_check || j * Bc + c < KV) && (HSK == HSK_pad || d < HSK / 4)) { + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + c, kcol); + if (kv_active && (HSK == HSK_pad || d < HSK / 4)) { #if !defined(BFLOAT16) if (USE_DECODE_K) { - uint coord = (j * Bc + c) * k_stride * BLOCK_SIZE_K + 4 * d; + uint coord = kcol * k_stride * BLOCK_SIZE_K + 4 * d; uint ib = coord / BLOCK_SIZE_K; uint iqs = (coord % BLOCK_SIZE_K); K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K); } else #endif { - K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + c) * k_stride / 4 + d]); + K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + kcol * k_stride / 4 + d]); } } @@ -266,7 +277,7 @@ void main() { if (SHMEM_STAGING == 0) { // For quants we always need to dequant into kvsh; for f16/bf16 we can load // directly from global memory when alignment / bounds allow it. - const bool stage_k = USE_DECODE_K || KV_bounds_check || d * 16 + 16 > HSK; + const bool stage_k = USE_DECODE_K || KV_bounds_check || USE_SPARSE || d * 16 + 16 > HSK; if (stage_k) { barrier(); [[unroll]] for (uint32_t idx = 0; idx < Bc * MatBr / 4; idx += gl_WorkGroupSize.x) { @@ -274,17 +285,19 @@ void main() { uint32_t row = (idx + tid) / (MatBr / 4); if (idx + tid < Bc * MatBr / 4) { FLOAT_TYPEV4 K_Tf = FLOAT_TYPEV4(0); - if ((!KV_bounds_check || j * Bc + row < KV) && (HSK == HSK_pad || d * 16 + col_vec * 4 < HSK)) { + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + row, kcol); + if (kv_active && (HSK == HSK_pad || d * 16 + col_vec * 4 < HSK)) { #if !defined(BFLOAT16) if (USE_DECODE_K) { - uint coord = (j * Bc + row) * k_stride * BLOCK_SIZE_K + d * 16 + col_vec * 4; + uint coord = kcol * k_stride * BLOCK_SIZE_K + d * 16 + col_vec * 4; uint ib = coord / BLOCK_SIZE_K; uint iqs = (coord % BLOCK_SIZE_K); K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K); } else #endif { - K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + row) * k_stride / 4 + d * 16 / 4 + col_vec]); + K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + kcol * k_stride / 4 + d * 16 / 4 + col_vec]); } } @@ -401,17 +414,19 @@ void main() { uint32_t c = (idx + tid) / (HSV_pad / 4); if (idx + gl_WorkGroupSize.x <= Bc * HSV_pad / 4 || c < Bc) { FLOAT_TYPEV4 V_Tf = FLOAT_TYPEV4(0); - if ((!KV_bounds_check || j * Bc + c < KV) && (HSV == HSV_pad || d < HSV / 4)) { + uint32_t v_row; + bool kv_active = fa_kv_index(j * Bc + c, v_row); + if (kv_active && (HSV == HSV_pad || d < HSV / 4)) { #if !defined(BFLOAT16) if (USE_DECODE_V) { - uint coord = (j * Bc + c) * v_stride * BLOCK_SIZE_V + 4 * d; + uint coord = v_row * v_stride * BLOCK_SIZE_V + 4 * d; uint ib = coord / BLOCK_SIZE_V; uint iqs = (coord % BLOCK_SIZE_V); V_Tf = dequantize4(ib, iqs, v_offset, BINDING_IDX_V); } else #endif { - V_Tf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + (j * Bc + c) * v_stride / 4 + d]); + V_Tf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + v_row * v_stride / 4 + d]); } } @@ -441,21 +456,22 @@ void main() { if (SHMEM_STAGING == 0) { // For quants we always preload via kvsh. For f16/bf16 we only preload when // alignment / bounds force it (otherwise we coopMatLoad direct from data_vv4). - const bool stage_v = USE_DECODE_V || KV_bounds_check; + const bool stage_v = USE_DECODE_V || KV_bounds_check || USE_SPARSE; if (stage_v) { [[unroll]] for (uint32_t i = 0; i < v_loads_per_thread; ++i) { const uint idx = i * gl_WorkGroupSize.x + tid; const uint row = idx / v_cols; const uint col = idx % v_cols; - const uint v_row = j * Bc + row; + uint32_t v_row; + bool kv_active = fa_kv_index(j * Bc + row, v_row); const uint v_col = hsv_tile * MatBc * row_split + col * 4; const uint coord = v_row * v_stride * BLOCK_SIZE_V + v_col; const uint ib = coord / BLOCK_SIZE_V; const uint iqs = coord % BLOCK_SIZE_V; - if (!KV_bounds_check || (v_row < KV && v_col < HSV)) { + if (USE_SPARSE ? (kv_active && v_col < HSV) : (!KV_bounds_check || (v_row < KV && v_col < HSV))) { #if !defined(BFLOAT16) if (USE_DECODE_V) { kvsh[row * vsh_stride + col] = dequantize4(ib, iqs, v_offset, BINDING_IDX_V); @@ -479,7 +495,7 @@ void main() { coopMatLoad(KMat, Psh, bc_chunk * MatBc * psh_stride, psh_stride, gl_CooperativeMatrixLayoutColumnMajor); if (SHMEM_STAGING == 0) { - if (!USE_DECODE_V && !KV_bounds_check) { + if (!USE_DECODE_V && !KV_bounds_check && !USE_SPARSE) { // F16/BF16 values can be loaded directly from global memory const uint v_tile_row = j * Bc + bc_chunk * MatBc; const uint v_tile_offset = v_offset / 4 + v_tile_row * v_stride / 4 + hsv_offset / 4; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp index 317411153087..c6ed63dd42a6 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp @@ -29,6 +29,12 @@ #include "dequant_funcs_cm2.glsl" #endif +#ifdef GL_NV_cooperative_matrix_decode_vector +#define FA_GATHER_BS 4u +#else +#define FA_GATHER_BS 1u +#endif + // buffer_reference stride = sizeof(struct) = FaBlockBytesK/V. layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufFA_K { uint8_t raw[FaBlockBytesK]; @@ -40,26 +46,28 @@ layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufFA_ #if !defined(BFLOAT16) float16_t faDecodeK(const decodeBufFA_K bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeK) { - case FA_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock); default: return float16_t(0); } } float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeV) { - case FA_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock); default: return float16_t(0); } } @@ -67,26 +75,26 @@ float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const // V=4 vector decode for K/V; dispatches to per-format _v decoders. f16vec4 faDecodeKVector(const decodeBufFA_K bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeK) { - case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); - case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); default: return f16vec4(0); } } f16vec4 faDecodeVVector(const decodeBufFA_V bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeV) { - case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); - case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); default: return f16vec4(0); } } @@ -105,6 +113,67 @@ layout (binding = 1) readonly buffer K {uint8_t data_k[];}; layout (binding = 2) readonly buffer V {uint8_t data_v[];}; layout (binding = 3) readonly buffer M {uint8_t data_m[];}; +// f16 aliases for the sparse gather callbacks. +layout (binding = 1) readonly buffer KF16 {float16_t data_kf16[];}; +layout (binding = 2) readonly buffer VF16 {float16_t data_vf16[];}; +layout (binding = 3) readonly buffer MF16 {float16_t data_mf16[];}; +#ifdef GL_NV_cooperative_matrix_decode_vector +layout (binding = 1) readonly buffer KF16V4 {f16vec4 data_kf16v4[];}; +layout (binding = 2) readonly buffer VF16V4 {f16vec4 data_vf16v4[];}; +#endif + +// K/V/mask f16-element offsets for the current head/batch, set in main(). +uint32_t g_k_off_elem, g_v_off_elem, g_m_off_elem; + +#if !defined(BFLOAT16) +// blockCoords are in block units: KV slot = blockCoords[0], +// head dim = blockCoords[1]*FA_GATHER_BS + coordInBlock[1]. +float16_t faGatherK(const decodeBufFA_K unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return float16_t(0); } + const int r = data_sparse[sparse_base + blockCoords[0]]; + return r < 0 ? float16_t(0) : data_kf16[g_k_off_elem + uint(r) * k_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]]; +} + +float16_t faGatherV(const decodeBufFA_V unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return float16_t(0); } + const int r = data_sparse[sparse_base + blockCoords[0]]; + return r < 0 ? float16_t(0) : data_vf16[g_v_off_elem + uint(r) * v_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]]; +} + +#ifdef GL_NV_cooperative_matrix_decode_vector +f16vec4 faGatherKVector(const decodeBufFA_K unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return f16vec4(0); } + const int r = data_sparse[sparse_base + blockCoords[0]]; + if (r < 0) { return f16vec4(0); } + const uint32_t o = g_k_off_elem + uint(r) * k_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]; + return data_kf16v4[o / 4]; +} + +f16vec4 faGatherVVector(const decodeBufFA_V unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return f16vec4(0); } + const int r = data_sparse[sparse_base + blockCoords[0]]; + if (r < 0) { return f16vec4(0); } + const uint32_t o = g_v_off_elem + uint(r) * v_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]; + return data_vf16v4[o / 4]; +} + +#define FAGATHERK , faGatherK, faGatherKVector +#define FAGATHERV , faGatherV, faGatherVVector +#else +#define FAGATHERK , faGatherK +#define FAGATHERV , faGatherV +#endif +#endif + +// Add gathered mask to S (slope==1 since sparse requires max_bias==0). col = slot in block jblk. +ACC_TYPE faAddSparseMask(const uint32_t row, const uint32_t col, const ACC_TYPE elem, const uint32_t jblk) { + const float NEG = uintBitsToFloat(0xFEFFFFFF); + const uint32_t kvslot = jblk * Bc + col; + if (kvslot >= p.split_kv) { return ACC_TYPE(NEG); } + const int r = data_sparse[sparse_base + kvslot]; + return r < 0 ? ACC_TYPE(NEG) : elem + ACC_TYPE(data_mf16[g_m_off_elem + row * m_stride + uint(r)]); +} + ACC_TYPE maxReduce(const in ACC_TYPE x, const in ACC_TYPE y) { return max(x, y); } @@ -183,14 +252,16 @@ void main() { tensorViewNV<2, false, 1, 0> tensorViewTranspose = createTensorViewNV(2, false, 1, 0); - const uint bs_k = fa_block_elems(FaTypeK); - const uint bs_v = fa_block_elems(FaTypeV); + const uint bs_k = USE_SPARSE ? FA_GATHER_BS : fa_block_elems(FaTypeK); + const uint bs_v = USE_SPARSE ? FA_GATHER_BS : fa_block_elems(FaTypeV); tensorLayoutK = setTensorLayoutBlockSizeNV(tensorLayoutK, 1, bs_k); tensorLayoutV = setTensorLayoutBlockSizeNV(tensorLayoutV, 1, bs_v); + // Sparse iterates n_kv_max (in split_kv); the decode callbacks remap each slot. + const uint32_t KV_iter = USE_SPARSE ? p.split_kv : KV; tensorLayoutQ = setTensorLayoutDimensionNV(tensorLayoutQ, N, HSK); - tensorLayoutK = setTensorLayoutDimensionNV(tensorLayoutK, KV, HSK); - tensorLayoutV = setTensorLayoutDimensionNV(tensorLayoutV, KV, HSV); + tensorLayoutK = setTensorLayoutDimensionNV(tensorLayoutK, KV_iter, HSK); + tensorLayoutV = setTensorLayoutDimensionNV(tensorLayoutV, KV_iter, HSV); // hint to the compiler that strides are aligned for the aligned variant of the shader if (Clamp != gl_CooperativeMatrixClampModeConstantNV) @@ -248,6 +319,10 @@ void main() { mo_offset += ((iq3 % p.nem3) * p.nem2 + (iq2 % p.nem2)) * CEIL_DIV(p.nem1, Br) * mo_stride; } + g_k_off_elem = (ik2*p.nb12 + ik3*p.nb13) / 2; + g_v_off_elem = (iv2*p.nb22 + iv3*p.nb23) / 2; + g_m_off_elem = m_offset / 2; + uint32_t mask_opt = 0; uint32_t mask_opt_idx = ~0; @@ -255,7 +330,7 @@ void main() { for (uint32_t j = start_j; j < end_j; ++j) { coopmat<float16_t, gl_ScopeWorkgroup, Br, Bc, gl_MatrixUseAccumulator> mv = coopmat<float16_t, gl_ScopeWorkgroup, Br, Bc, gl_MatrixUseAccumulator>(0); - if (MASK_ENABLE) { + if (MASK_ENABLE && !USE_SPARSE) { if (USE_MASK_OPT && mask_opt_idx != j / 16) { mask_opt_idx = j / 16; @@ -313,7 +388,9 @@ void main() { coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose); #else const bool k_use_decode = (bs_k > 1u); - if (k_use_decode) { + if (USE_SPARSE) { + coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose FAGATHERK); + } else if (k_use_decode) { coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose FADECODEK); } else { coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose); @@ -328,7 +405,9 @@ void main() { } } - if (MASK_ENABLE) { + if (MASK_ENABLE && USE_SPARSE) { + coopMatPerElementNV(S, S, faAddSparseMask, j); + } else if (MASK_ENABLE) { S += slopeMat*coopmat<ACC_TYPE, gl_ScopeWorkgroup, Br, Bc, gl_MatrixUseAccumulator>(mv); } @@ -383,7 +462,9 @@ void main() { coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad)); #else const bool v_use_decode = (bs_v > 1u); - if (v_use_decode) { + if (USE_SPARSE) { + coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad) FAGATHERV); + } else if (v_use_decode) { coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad) FADECODEV); } else { coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_1.comp new file mode 100644 index 000000000000..b5f95aaa0d5f --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_1.comp @@ -0,0 +1,263 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : enable +#extension GL_EXT_shader_16bit_storage : require +#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require +#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require +#extension GL_KHR_memory_scope_semantics : enable +#extension GL_KHR_shader_subgroup_basic : enable +#extension GL_KHR_shader_subgroup_ballot : enable +#extension GL_KHR_shader_subgroup_arithmetic : enable +#extension GL_KHR_cooperative_matrix : enable +#extension GL_EXT_shared_memory_block : enable + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer Q {float16_t qState[];}; +layout (binding = 1) readonly buffer K_VEC4 {f16vec4 kStateVec4[];}; +layout (binding = 2) buffer MASK_F16 {float16_t mState_f16[];}; +layout (binding = 3) buffer P_FP16 {float16_t matP_f16[];}; +layout (binding = 4) buffer OUT_MAX {float out_max_f32[];}; + +layout (push_constant) uniform parameter +{ + uint kvSeqLen; + uint activationLength; + uint qHead; + uint kvHead; + uint qkRatio; + uint qkSubGroups; + uint flag; + uint kvStride1; + uint kvStride2; + uint batchStrideQ; + uint batchStrideK; + uint batchStrideV; + uint batchStrideM; + uint batchStrideO; + float softMaxScale; +} p; + +layout (constant_id = 0) const uint GROUPSIZE = 128; +layout (constant_id = 1) const uint GQA_RATIO = 8; +layout (constant_id = 2) const uint HEAD_DIM = 128; +layout (constant_id = 3) const uint WARPSIZE = 16; +layout (constant_id = 4) const uint MATP_REDUCE = 32; +layout (constant_id = 5) const uint N_TOK = 1; +layout (constant_id = 6) const uint COOP_MAT_P_PER_LOOP = 4; + +#define MAX_HEADS 8 + +#define TN WARPSIZE +#define TM 8 +#define TK 16 +#define SUBGROUP_COUNT (GROUPSIZE / WARPSIZE) +#define MATP_PER_LOOP (COOP_MAT_P_PER_LOOP * TM) +#define P_LOOP_COUNT (MATP_REDUCE / MATP_PER_LOOP) + +#define COOP_MAT_Q_PER_TOKEN ((GQA_RATIO + TN - 1) / TN) +#define COOP_MAT_P_M COOP_MAT_Q_PER_TOKEN +#define COOP_MAT_P_N (MATP_REDUCE / TM) +#define SLM_PV_SIZE (MATP_REDUCE * COOP_MAT_P_M * TN) +#define SLM_MASK_SIZE (N_TOK * MATP_REDUCE) +#define SLM_POOL_SIZE_K (MATP_PER_LOOP * HEAD_DIM) +#define K_LOAD_PER_LOOP (GROUPSIZE * 4) +#define HEAD_DIM_VEC4 (HEAD_DIM / 4) +#define SLM_CHUNK_SIZE (TK / 4) +#define K_LOAD_LOOPS ((SLM_POOL_SIZE_K + K_LOAD_PER_LOOP - 1) / K_LOAD_PER_LOOP) +#define O_COUNT ((GQA_RATIO + SUBGROUP_COUNT - 1) / SUBGROUP_COUNT) + +shared slm_pool_block { + float slm_pool_pv[SLM_PV_SIZE + SLM_MASK_SIZE]; +} slm_pool_f32; + +shared slm_pool_alias_block { + float16_t slm_pool_k[SLM_POOL_SIZE_K]; +} slm_pool_f16; + +void main() { + const uint lane = gl_SubgroupInvocationID; + const uint kHeadIdx = gl_WorkGroupID.x % p.kvHead; + const uint outGroupIdx = gl_WorkGroupID.x / p.kvHead; + const uint v = gl_WorkGroupID.y; + const uint d = gl_WorkGroupID.z; + const uint localLinearId = gl_SubgroupID; + const uint wgLane = localLinearId * WARPSIZE + lane; + const uint qDim = p.qHead * HEAD_DIM; + const uint kvDim = p.kvStride1; + const uint maskDim = p.kvSeqLen; + const uint maxDim = (p.kvSeqLen + MATP_REDUCE - 1) / MATP_REDUCE; + const uint pDim = maxDim * MATP_REDUCE; + const uint tokFlatIdx = localLinearId + outGroupIdx * N_TOK; + uint offsetBaseQ = min(tokFlatIdx, p.activationLength - 1) * qDim; + offsetBaseQ = offsetBaseQ + d * p.batchStrideQ + kHeadIdx * HEAD_DIM * GQA_RATIO; + const uint offsetBaseK = (d * p.batchStrideK + (v * MATP_REDUCE) * kvDim + kHeadIdx * p.kvStride2) / 4; + uint offsetOut = d * p.qHead * p.activationLength * pDim + v * MATP_REDUCE + kHeadIdx * GQA_RATIO * pDim + (localLinearId * O_COUNT + outGroupIdx * N_TOK * p.qHead) * pDim + lane; + uint offsetMax = d * p.qHead * p.activationLength * maxDim + v + kHeadIdx * GQA_RATIO * maxDim + (localLinearId * O_COUNT + outGroupIdx * N_TOK * p.qHead) * maxDim; + const uint offsetSlmLoadPv = (localLinearId * O_COUNT * MATP_REDUCE + lane); + const uint offsetBaseM = v * MATP_REDUCE + lane; + const float fp32Min = uintBitsToFloat(0xFEFFFFFF); + + const uint loopCount = HEAD_DIM / TK; + float maskFp32[MATP_REDUCE / WARPSIZE]; + + if (tokFlatIdx < p.activationLength) { + [[unroll]] for (uint mk = 0; mk < MATP_REDUCE / WARPSIZE; mk++) { + const uint maskOffset = mk * WARPSIZE + offsetBaseM; + if (maskOffset < maskDim) { + maskFp32[mk] = float(mState_f16[d * p.batchStrideM + tokFlatIdx * maskDim + maskOffset]); + } else { + maskFp32[mk] = fp32Min; + } + } + } + + coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> matP[COOP_MAT_P_M][COOP_MAT_P_N]; + + [[unroll]] for (uint mp = 0; mp < COOP_MAT_P_M; mp++) { + [[unroll]] for (uint np = 0; np < COOP_MAT_P_N; np++) { + matP[mp][np] = coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(0.0f); + } + } + + [[unroll]] for (uint kLoad = 0; kLoad < K_LOAD_LOOPS; kLoad++) { + const uint flatOffset = kLoad * GROUPSIZE + wgLane; + const uint kRowIdx = flatOffset / HEAD_DIM_VEC4; + const uint kColIdx = flatOffset % HEAD_DIM_VEC4; + const uint slmChunkCol = kColIdx % SLM_CHUNK_SIZE; + const uint slmChunkRow = kColIdx / SLM_CHUNK_SIZE; + const uint offsetK = offsetBaseK + kRowIdx * kvDim / 4 + kColIdx; + const uint offsetSlmK = kRowIdx * TK + slmChunkRow * TK * MATP_PER_LOOP + slmChunkCol * 4; + slm_pool_f16.slm_pool_k[offsetSlmK + 0] = kStateVec4[offsetK].x; + slm_pool_f16.slm_pool_k[offsetSlmK + 1] = kStateVec4[offsetK].y; + slm_pool_f16.slm_pool_k[offsetSlmK + 2] = kStateVec4[offsetK].z; + slm_pool_f16.slm_pool_k[offsetSlmK + 3] = kStateVec4[offsetK].w; + } + + [[unroll]] for (uint pLoop = 0; pLoop < P_LOOP_COUNT; pLoop++) { + f16vec4 kTemp[K_LOAD_LOOPS]; + + if (pLoop + 1 < P_LOOP_COUNT) { + [[unroll]] for (uint kLoad = 0; kLoad < K_LOAD_LOOPS; kLoad++) { + const uint flatOffset = kLoad * GROUPSIZE + wgLane; + const uint kRowIdx = flatOffset / HEAD_DIM_VEC4 + (pLoop + 1) * MATP_PER_LOOP; + const uint kColIdx = flatOffset % HEAD_DIM_VEC4; + const uint offsetK = offsetBaseK + kRowIdx * kvDim / 4 + kColIdx; + kTemp[kLoad] = kStateVec4[offsetK]; + } + } + + barrier(); + if (localLinearId < N_TOK) { + [[unroll]] for (uint loop = 0; loop < loopCount; loop++) { + coopmat<float16_t, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> matQ[COOP_MAT_P_M]; + coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> matK[COOP_MAT_P_PER_LOOP]; + + [[unroll]] for (uint mq = 0; mq < COOP_MAT_P_M; mq++) { + coopMatLoad( + matQ[mq], + qState, + offsetBaseQ + mq * TN * HEAD_DIM + loop * TK, + HEAD_DIM, + gl_CooperativeMatrixLayoutColumnMajor); + } + + [[unroll]] for (uint np = 0; np < COOP_MAT_P_PER_LOOP; np++) { + coopMatLoad( + matK[np], + slm_pool_f16.slm_pool_k, + loop * TK * MATP_PER_LOOP + np * TM * TK, + TK, + gl_CooperativeMatrixLayoutRowMajor); + } + + [[unroll]] for (uint mp = 0; mp < COOP_MAT_P_M; mp++) { + [[unroll]] for (uint np = 0; np < COOP_MAT_P_PER_LOOP; np++) { + matP[mp][pLoop * COOP_MAT_P_PER_LOOP + np] = coopMatMulAdd(matK[np], matQ[mp], matP[mp][pLoop * COOP_MAT_P_PER_LOOP + np]); + } + } + } + } + + barrier(); + + if (pLoop + 1 < P_LOOP_COUNT) { + [[unroll]] for (uint kLoad = 0; kLoad < K_LOAD_LOOPS; kLoad++) { + const uint flatOffset = kLoad * GROUPSIZE + wgLane; + const uint kRowIdx = flatOffset / HEAD_DIM_VEC4; + const uint kColIdx = flatOffset % HEAD_DIM_VEC4; + const uint slmChunkCol = kColIdx % SLM_CHUNK_SIZE; + const uint slmChunkRow = kColIdx / SLM_CHUNK_SIZE; + const uint offsetSlmK = kRowIdx * TK + slmChunkRow * TK * MATP_PER_LOOP + slmChunkCol * 4; + slm_pool_f16.slm_pool_k[offsetSlmK + 0] = kTemp[kLoad].x; + slm_pool_f16.slm_pool_k[offsetSlmK + 1] = kTemp[kLoad].y; + slm_pool_f16.slm_pool_k[offsetSlmK + 2] = kTemp[kLoad].z; + slm_pool_f16.slm_pool_k[offsetSlmK + 3] = kTemp[kLoad].w; + } + } + } + + barrier(); + + if (tokFlatIdx < p.activationLength) { + [[unroll]] for (uint mk = 0; mk < MATP_REDUCE / WARPSIZE; mk++) { + slm_pool_f32.slm_pool_pv[SLM_PV_SIZE + localLinearId * MATP_REDUCE + mk * WARPSIZE + lane] = maskFp32[mk]; + } + } + + [[unroll]] for (uint oLoop = 0; oLoop < N_TOK; oLoop++) { + if (oLoop + outGroupIdx * N_TOK < p.activationLength) { + if (localLinearId == oLoop) { + [[unroll]] for (uint mp = 0; mp < COOP_MAT_P_M; mp++) { + [[unroll]] for (uint np = 0; np < COOP_MAT_P_N; np++) { + coopMatStore(matP[mp][np], slm_pool_f32.slm_pool_pv, mp * MATP_REDUCE * TN + np * TM, MATP_REDUCE, gl_CooperativeMatrixLayoutColumnMajor); + } + } + } + + barrier(); + + [[unroll]] for (uint maskIdx = 0; maskIdx < MATP_REDUCE / WARPSIZE; maskIdx++) { + maskFp32[maskIdx] = slm_pool_f32.slm_pool_pv[SLM_PV_SIZE + oLoop * MATP_REDUCE + maskIdx * WARPSIZE + lane]; + } + + float fp32O[O_COUNT][MATP_REDUCE / WARPSIZE]; + float maxOut[O_COUNT]; + + [[unroll]] for (uint oc = 0; oc < O_COUNT; oc++) { + [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { + fp32O[oc][os] = slm_pool_f32.slm_pool_pv[offsetSlmLoadPv + os * WARPSIZE + oc * MATP_REDUCE] * p.softMaxScale; + } + + [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { + fp32O[oc][os] = fp32O[oc][os] + maskFp32[os]; + } + + float maxTemp = fp32Min; + [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { + maxTemp = max(maxTemp, fp32O[oc][os]); + } + maxOut[oc] = subgroupMax(maxTemp); + [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { + fp32O[oc][os] = exp(fp32O[oc][os] - maxOut[oc]); + } + } + + [[unroll]] for (uint oc = 0; oc < O_COUNT; oc++) { + if (localLinearId * O_COUNT + oc < GQA_RATIO) { + [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { + matP_f16[offsetOut + oc * pDim + os * WARPSIZE] = float16_t(fp32O[oc][os]); + } + + if (lane == 0) { + out_max_f32[offsetMax + oc * maxDim] = maxOut[oc]; + } + } + } + + offsetOut = offsetOut + p.qHead * pDim; + offsetMax = offsetMax + p.qHead * maxDim; + barrier(); + } + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_2.comp new file mode 100644 index 000000000000..60a1c2ce7362 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_2.comp @@ -0,0 +1,408 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : enable +#extension GL_EXT_shader_16bit_storage : require +#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require +#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require +#extension GL_KHR_memory_scope_semantics : enable +#extension GL_KHR_shader_subgroup_basic : enable +#extension GL_KHR_shader_subgroup_ballot : enable +#extension GL_KHR_shader_subgroup_arithmetic : enable +#extension GL_KHR_cooperative_matrix : enable +#extension GL_EXT_shared_memory_block : enable + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer P {f16vec4 pStateVec4[];}; +layout (binding = 1) readonly buffer V {float16_t vState[];}; +layout (binding = 1) readonly buffer V_VEC4 {f16vec4 vStateVec4[];}; +layout (binding = 2) buffer MAX_FP32 {float max_f32[];}; +layout (binding = 3) buffer SINK_FP32 {float sink_f32[];}; +layout (binding = 4) buffer OUT_FP32 {float out_f32[];}; +layout (binding = 4) buffer OUT_VEC4 {vec4 out_f32_vec4[];}; +layout (binding = 4) buffer OUT_F16 {float16_t out_f16[];}; + +layout (push_constant) uniform parameter +{ + uint kvSeqLen; + uint activationLength; + uint qHead; + uint kvHead; + uint qkRatio; + uint qkSubGroups; + uint flag; + uint kvStride1; + uint kvStride2; + uint batchStrideQ; + uint batchStrideK; + uint batchStrideV; + uint batchStrideM; + uint batchStrideO; + float softMaxScale; +} p; + +layout (constant_id = 0) const uint GROUPSIZE = 256; +layout (constant_id = 1) const uint GQA_RATIO = 8; +layout (constant_id = 2) const uint HEAD_DIM = 128; +layout (constant_id = 3) const uint N_TOKS_PER_GROUP = 1; +layout (constant_id = 4) const uint WARPSIZE = 16; +layout (constant_id = 5) const uint MATP_PER_LOOP = 64; +layout (constant_id = 6) const uint MATP_REDUCE = 32; +layout (constant_id = 7) const uint WARP_V_DIM = 16; + +#define TN WARPSIZE +#define TM 8 +#define TK 16 +#define MAT_O_N (WARP_V_DIM / TM) +#define MAT_P_M (GQA_RATIO * N_TOKS_PER_GROUP) +#define ALIGNED_P_M ((MAT_P_M + WARPSIZE - 1) / WARPSIZE) +#define V_HEAD_GROUPS (HEAD_DIM / WARP_V_DIM) + +#define SUBGROUP_COUNT (GROUPSIZE / WARPSIZE) +#define SPLIT_P_GROUPS (MATP_PER_LOOP / TK) + +#define SLM_POOL_SIZE_O (SUBGROUP_COUNT * ALIGNED_P_M * TN * MAT_O_N * TM) + +#define P_LOAD_PER_LOOP (GROUPSIZE * 4) +#define P_LOAD_LOOPS ((MAT_P_M * MATP_PER_LOOP + P_LOAD_PER_LOOP - 1) / P_LOAD_PER_LOOP) +#define SLM_POOL_SIZE_P (P_LOAD_LOOPS * P_LOAD_PER_LOOP) +#define SIZE_LOCAL_MAX (MAT_P_M * MATP_PER_LOOP / MATP_REDUCE) +#define MAX_LOAD_LOOPS ((SIZE_LOCAL_MAX + GROUPSIZE - 1) / GROUPSIZE) +#define SLM_POOL_SIZE_LOCAL_MAX (MAX_LOAD_LOOPS * GROUPSIZE) +#define MAX_REDUCE_COUNT ((MAT_P_M + SUBGROUP_COUNT - 1) / SUBGROUP_COUNT) +#define GLOBAL_MAX_SIZE (MAX_REDUCE_COUNT * SUBGROUP_COUNT) + +#define SLM_POOL_SIZE_SOFTMAX_SUM (SUBGROUP_COUNT * P_LOAD_LOOPS) + +#define SLM_OFFSET_P (GLOBAL_MAX_SIZE * 2 + SLM_POOL_SIZE_SOFTMAX_SUM * 2 + SLM_POOL_SIZE_LOCAL_MAX * 2 * 2) + +#define SLM_OFFSET_GLOBAL_MAX 0 +#define SLM_OFFSET_SOFTMAX_SUM (SLM_OFFSET_GLOBAL_MAX + GLOBAL_MAX_SIZE) +#define SLM_OFFSET_O (SLM_OFFSET_SOFTMAX_SUM + SLM_POOL_SIZE_SOFTMAX_SUM) +#define SLM_OFFSET_LOCAL_MAX (GLOBAL_MAX_SIZE + SLM_POOL_SIZE_SOFTMAX_SUM) + +#define P_REDUCE_VEC4 (MATP_PER_LOOP / 4) +#define MAX_PER_LOOP (MATP_PER_LOOP / MATP_REDUCE) +#define SLM_MAX_STRIDE (MATP_REDUCE / 4) +#define SUB_GROUPS_PER_LINE (MATP_PER_LOOP / WARPSIZE / 4) + +shared slm_pool_block { + float slm_pool_o[GLOBAL_MAX_SIZE + SLM_POOL_SIZE_SOFTMAX_SUM + SLM_POOL_SIZE_O]; +} slm_pool_f32; + +shared slm_pool_alias_block { + float16_t slm_pool_pv[GLOBAL_MAX_SIZE * 2 + SLM_POOL_SIZE_SOFTMAX_SUM * 2 + SLM_POOL_SIZE_LOCAL_MAX * 2 * 2 + SLM_POOL_SIZE_P * 2]; +} slm_pool_alias_f16; + +void main() { + const uint lane = gl_SubgroupInvocationID; + const uint v = gl_WorkGroupID.y; + const uint d = gl_WorkGroupID.z; + const uint vWarpIdx = gl_WorkGroupID.x % V_HEAD_GROUPS; + const uint outTokIdx = gl_WorkGroupID.x / V_HEAD_GROUPS; + const uint localLinearId = gl_SubgroupID; + const uint wgLane = localLinearId * WARPSIZE + lane; + const uint splitIdx = localLinearId; + const uint maxDim = (p.kvSeqLen + MATP_REDUCE - 1) / MATP_REDUCE; + const uint pDim = maxDim * MATP_REDUCE; + const uint kvDim = p.kvStride1; + const uint oDim = p.qHead * HEAD_DIM; + const uint offsetBaseP = (d * p.activationLength * p.qHead + v * GQA_RATIO + outTokIdx * N_TOKS_PER_GROUP * p.qHead) * pDim / 4; + const uint offsetBaseMax = (d * p.activationLength * p.qHead + v * GQA_RATIO + outTokIdx * N_TOKS_PER_GROUP * p.qHead) * maxDim; + const uint offsetBaseV = (d * p.batchStrideV + v * p.kvStride2 + vWarpIdx * WARP_V_DIM + splitIdx * TK * kvDim); + const uint offsetSlmP = (SLM_OFFSET_P + wgLane * 4); + const float fp32Min = uintBitsToFloat(0xFEFFFFFF); + const float fp32Max = uintBitsToFloat(0x7EFFFFFF); + uint offsetV = offsetBaseV; + + coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> sums[ALIGNED_P_M][MAT_O_N]; + f16vec4 pStateTemp[P_LOAD_LOOPS]; + + float fp32CompensationP[P_LOAD_LOOPS]; + + uint loadRowBase[P_LOAD_LOOPS]; + uint loadColBase[P_LOAD_LOOPS]; + float fp32SoftMaxSum[P_LOAD_LOOPS]; + float fp32GlobalMaxP[P_LOAD_LOOPS]; + uint maxRowBase[MAX_LOAD_LOOPS]; + uint maxColBase[MAX_LOAD_LOOPS]; + uint outOffsets[ALIGNED_P_M]; + bool outputMask[ALIGNED_P_M]; + float fp32SinkCoeff[ALIGNED_P_M]; + + [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { + const uint flatOffset = pm * WARPSIZE + lane; + const uint inGroupTokIdx = flatOffset / GQA_RATIO; + const uint inGroupHeadIdx = flatOffset % GQA_RATIO; + outputMask[pm] = (N_TOKS_PER_GROUP * outTokIdx + inGroupTokIdx < p.activationLength) && (inGroupHeadIdx < GQA_RATIO) && (inGroupTokIdx < N_TOKS_PER_GROUP); + outOffsets[pm] = (inGroupTokIdx * oDim + inGroupHeadIdx * HEAD_DIM) / 4; + if ((0x1 & p.flag) != 0) { + fp32SinkCoeff[pm] = sink_f32[inGroupHeadIdx + v * GQA_RATIO]; + } + } + + [[unroll]] for (uint maxCount = 0; maxCount < MAX_REDUCE_COUNT; maxCount++) { + const uint flatIdx = maxCount * SUBGROUP_COUNT + localLinearId; + const uint rowIdx = flatIdx % GQA_RATIO; + const uint tokIdx = flatIdx / GQA_RATIO; + + if (tokIdx < N_TOKS_PER_GROUP) { + float fp32MaxReduce = fp32Min; + const uint maxOffset = offsetBaseMax + (tokIdx * p.qHead + rowIdx) * maxDim; + [[unroll]] for (uint maxReduce = 0; maxReduce < (maxDim + WARPSIZE - 1) / WARPSIZE; maxReduce++) { + if (maxReduce * WARPSIZE + lane < maxDim) { + fp32MaxReduce = max(fp32MaxReduce, max_f32[maxOffset + maxReduce * WARPSIZE + lane]); + } + } + fp32MaxReduce = subgroupMax(fp32MaxReduce); + if (lane == 0) { + slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + maxCount * SUBGROUP_COUNT + localLinearId] = fp32MaxReduce; + } + } else { + if (lane == 0) { + slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + maxCount * SUBGROUP_COUNT + localLinearId] = fp32Max; + } + } + } + + barrier(); + + [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { + const uint flatOffset = (pLoad * GROUPSIZE + wgLane) / P_REDUCE_VEC4; + const uint rowIdxFlat = flatOffset % GQA_RATIO; + const uint tokenIdxFlat = min(flatOffset / GQA_RATIO, N_TOKS_PER_GROUP - 1); + loadColBase[pLoad] = (pLoad * GROUPSIZE + wgLane) % P_REDUCE_VEC4; + loadRowBase[pLoad] = (tokenIdxFlat * p.qHead + rowIdxFlat); + fp32SoftMaxSum[pLoad] = 0.0f; + fp32GlobalMaxP[pLoad] = slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + flatOffset]; + } + + [[unroll]] for (uint maxLoad = 0; maxLoad < MAX_LOAD_LOOPS; maxLoad++) { + const uint flatOffset = (maxLoad * GROUPSIZE + wgLane) / MAX_PER_LOOP; + const uint rowIdxFlat = flatOffset % GQA_RATIO; + const uint tokenIdxFlat = min(flatOffset / GQA_RATIO, N_TOKS_PER_GROUP - 1); + maxColBase[maxLoad] = (maxLoad * GROUPSIZE + wgLane) % MAX_PER_LOOP; + maxRowBase[maxLoad] = (tokenIdxFlat * p.qHead + rowIdxFlat); + } + + [[unroll]] for (uint maxLoad = 0; maxLoad < MAX_LOAD_LOOPS; maxLoad++) { + const uint flatMaxOffset = maxRowBase[maxLoad] * maxDim + maxColBase[maxLoad]; + slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + maxLoad * GROUPSIZE + wgLane] = max_f32[offsetBaseMax + flatMaxOffset]; + maxColBase[maxLoad] = maxColBase[maxLoad] + MATP_PER_LOOP / MATP_REDUCE; + } + + [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { + const uint flatOffset = loadRowBase[pLoad] * pDim / 4 + loadColBase[pLoad]; + pStateTemp[pLoad] = pStateVec4[offsetBaseP + flatOffset]; + } + + [[unroll]] for (uint n = 0; n < ALIGNED_P_M; n++) { + [[unroll]] for (uint i = 0; i < MAT_O_N; i++) { + sums[n][i] = coopmat<float, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(0.0f); + } + } + + barrier(); + + [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { + const uint maxOffset = (pLoad * GROUPSIZE + wgLane) / SLM_MAX_STRIDE; + if (loadColBase[pLoad] < pDim / 4) { + fp32CompensationP[pLoad] = slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + maxOffset]; + float pTemp[4] = float[4](pStateTemp[pLoad].x, pStateTemp[pLoad].y, pStateTemp[pLoad].z, pStateTemp[pLoad].w); + float compTemp = exp(fp32CompensationP[pLoad] - fp32GlobalMaxP[pLoad]); + [[unroll]] for (uint kk = 0; kk < 4; kk++) { + pTemp[kk] = pTemp[kk] * compTemp; + fp32SoftMaxSum[pLoad] = fp32SoftMaxSum[pLoad] + pTemp[kk]; + slm_pool_alias_f16.slm_pool_pv[offsetSlmP + pLoad * GROUPSIZE * 4 + kk] = float16_t(pTemp[kk]); + } + } else { + [[unroll]] for (uint kk = 0; kk < 4; kk++) { + slm_pool_alias_f16.slm_pool_pv[offsetSlmP + pLoad * GROUPSIZE * 4 + kk] = float16_t(0.0f); + } + } + + loadColBase[pLoad] = loadColBase[pLoad] + P_REDUCE_VEC4; + } + + const uint loopCount = (p.kvSeqLen + MATP_PER_LOOP - 1) / MATP_PER_LOOP; + + for (uint loop = 0; loop < loopCount; loop++) { + const uint slmPingPongLoad = (loop & 0x1); + const uint slmPingPongStore = ((loop + 1) & 0x1); + + if (loop + 1 < loopCount) { + [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { + const uint flatOffset = loadRowBase[pLoad] * pDim / 4 + loadColBase[pLoad]; + pStateTemp[pLoad] = pStateVec4[offsetBaseP + flatOffset]; + } + + [[unroll]] for (uint maxLoad = 0; maxLoad < MAX_LOAD_LOOPS; maxLoad++) { + const uint flatMaxOffset = maxRowBase[maxLoad] * maxDim + maxColBase[maxLoad]; + slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + slmPingPongStore * SLM_POOL_SIZE_LOCAL_MAX + maxLoad * GROUPSIZE + wgLane] = max_f32[offsetBaseMax + flatMaxOffset]; + maxColBase[maxLoad] = maxColBase[maxLoad] + MATP_PER_LOOP / MATP_REDUCE; + } + } + + barrier(); + + { + const uint coopMatOffsetP = SLM_OFFSET_P + slmPingPongLoad * SLM_POOL_SIZE_P + splitIdx * TK; + coopmat<float16_t, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> matV[MAT_O_N]; + [[unroll]] for (uint cc = 0; cc < MAT_O_N; cc++) { + coopMatLoad( + matV[cc], + vState, + offsetV + TM * cc, + kvDim, + gl_CooperativeMatrixLayoutColumnMajor); + } + [[unroll]] for (uint mo = 0; mo < ALIGNED_P_M; mo++) { + coopmat<float16_t, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> matP; + coopMatLoad( + matP, + slm_pool_alias_f16.slm_pool_pv, + coopMatOffsetP + mo * TN * MATP_PER_LOOP, + MATP_PER_LOOP, + gl_CooperativeMatrixLayoutColumnMajor); + + [[unroll]] for (uint no = 0; no < MAT_O_N; no++) { + sums[mo][no] = coopMatMulAdd(matV[no], matP, sums[mo][no]); + } + } + } + + offsetV += MATP_PER_LOOP * kvDim; + if (loop * MATP_PER_LOOP + splitIdx * TK >= p.kvSeqLen) { + offsetV = 0; + } + if (loop + 1 < loopCount) { + [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { + const uint maxOffset = (pLoad * GROUPSIZE + wgLane) / SLM_MAX_STRIDE; + if (loadColBase[pLoad] < pDim / 4) { + fp32CompensationP[pLoad] = slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + slmPingPongStore * SLM_POOL_SIZE_LOCAL_MAX + maxOffset]; + float pTemp[4] = float[4](pStateTemp[pLoad].x, pStateTemp[pLoad].y, pStateTemp[pLoad].z, pStateTemp[pLoad].w); + float compTemp = exp(fp32CompensationP[pLoad] - fp32GlobalMaxP[pLoad]); + [[unroll]] for (uint kk = 0; kk < 4; kk++) { + pTemp[kk] = pTemp[kk] * compTemp; + fp32SoftMaxSum[pLoad] = fp32SoftMaxSum[pLoad] + pTemp[kk]; + slm_pool_alias_f16.slm_pool_pv[offsetSlmP + slmPingPongStore * SLM_POOL_SIZE_P + pLoad * GROUPSIZE * 4 + kk] = float16_t(pTemp[kk]); + } + } else { + [[unroll]] for (uint kk = 0; kk < 4; kk++) { + slm_pool_alias_f16.slm_pool_pv[offsetSlmP + slmPingPongStore * SLM_POOL_SIZE_P + pLoad * GROUPSIZE * 4 + kk] = float16_t(0.0f); + } + } + loadColBase[pLoad] = loadColBase[pLoad] + P_REDUCE_VEC4; + } + } + } + + barrier(); + + [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { + fp32SoftMaxSum[pLoad] = subgroupAdd(fp32SoftMaxSum[pLoad]); + } + + [[unroll]] for (uint mo = 0; mo < ALIGNED_P_M; mo++) { + [[unroll]] for (uint no = 0; no < MAT_O_N; no++) { + coopMatStore( + sums[mo][no], + slm_pool_f32.slm_pool_o, + SLM_OFFSET_O + mo * TN * WARP_V_DIM + TM * no + localLinearId * ALIGNED_P_M * TN * WARP_V_DIM, + WARP_V_DIM, + gl_CooperativeMatrixLayoutColumnMajor); + } + } + + [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { + slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pLoad * SUBGROUP_COUNT + localLinearId] = fp32SoftMaxSum[pLoad]; + } + + barrier(); + + if (localLinearId == 1) { + const uint sumBase = SLM_OFFSET_SOFTMAX_SUM + lane * SUB_GROUPS_PER_LINE; + float sumTemp[ALIGNED_P_M][SUB_GROUPS_PER_LINE]; + [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { + [[unroll]] for (uint reduce = 0; reduce < SUB_GROUPS_PER_LINE; reduce++) { + sumTemp[pm][reduce] = slm_pool_f32.slm_pool_o[sumBase + pm * WARPSIZE * SUB_GROUPS_PER_LINE + reduce]; + } + } + + [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { + [[unroll]] for (uint reduce = 1; reduce < SUB_GROUPS_PER_LINE; reduce++) { + sumTemp[pm][0] = sumTemp[pm][0] + sumTemp[pm][reduce]; + } + } + + if ((0x1 & p.flag) != 0) { + [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { + float fp32GlobalMax = slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + pm * WARPSIZE + lane]; + float sinkCompensation = fp32GlobalMax - fp32SinkCoeff[pm]; + sinkCompensation = exp(sinkCompensation); + float softmaxSumTemp = sumTemp[pm][0] * sinkCompensation; + sumTemp[pm][0] = sumTemp[pm][0] + 1.0f / sinkCompensation; + sumTemp[pm][0] = 1.0f / sumTemp[pm][0]; + sinkCompensation = sinkCompensation / (1.0f + softmaxSumTemp); + sumTemp[pm][0] = fp32GlobalMax < fp32SinkCoeff[pm] ? sinkCompensation : sumTemp[pm][0]; + slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pm * WARPSIZE + lane] = sumTemp[pm][0]; + } + } else { + [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { + slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pm * WARPSIZE + lane] = 1.0f / sumTemp[pm][0]; + } + } + } + + [[unroll]] for (uint reduce = 2; reduce < SPLIT_P_GROUPS; reduce = reduce << 1 ) { + const uint stride = (reduce >> 1) * ALIGNED_P_M * TN * MAT_O_N * TM; + if ((localLinearId % reduce) == 0) { + const uint reduceBase = localLinearId * ALIGNED_P_M * TN * MAT_O_N * TM + SLM_OFFSET_O; + float sumTemp0[4]; + float sumTemp1[4]; + const uint reduceVec4Count = ALIGNED_P_M * TN * MAT_O_N * TM / 4 / WARPSIZE; + [[unroll]] for (uint totalLoads = 0; totalLoads < reduceVec4Count; totalLoads++) { + [[unroll]] for (uint kk = 0; kk < 4; kk++) { + sumTemp0[kk] = slm_pool_f32.slm_pool_o[reduceBase + totalLoads * 4 * WARPSIZE + 4 * lane + kk]; + sumTemp1[kk] = slm_pool_f32.slm_pool_o[reduceBase + stride + totalLoads * 4 * WARPSIZE + 4 * lane + kk]; + } + + [[unroll]] for (uint kk = 0; kk < 4; kk++) { + sumTemp0[kk] = sumTemp0[kk] + sumTemp1[kk]; + } + + [[unroll]] for (uint kk = 0; kk < 4; kk++) { + slm_pool_f32.slm_pool_o[reduceBase + totalLoads * 4 * WARPSIZE + 4 * lane + kk] = sumTemp0[kk]; + } + } + } + barrier(); + } + + if (localLinearId == 0) { + const uint slmBase0 = SLM_OFFSET_O + lane * WARP_V_DIM; + const uint slmBase1 = slmBase0 + SPLIT_P_GROUPS / 2 * ALIGNED_P_M * TN * MAT_O_N * TM; + + const uint offsetOutBase = (d * p.batchStrideO + vWarpIdx * WARP_V_DIM + v * GQA_RATIO * HEAD_DIM + outTokIdx * oDim * N_TOKS_PER_GROUP) / 4; + float fp32SoftMaxMul[ALIGNED_P_M]; + float fp32Output[ALIGNED_P_M][4]; + [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { + fp32SoftMaxMul[pm] = slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pm * WARPSIZE + lane]; + } + + [[unroll]] for (uint vg = 0; vg < WARP_V_DIM / 4; vg++) { + [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { + [[unroll]] for (uint vc = 0; vc < 4; vc++) { + fp32Output[pm][vc] = slm_pool_f32.slm_pool_o[slmBase0 + pm * WARPSIZE * WARP_V_DIM + vg * 4 + vc] * fp32SoftMaxMul[pm]; + fp32Output[pm][vc] = fp32Output[pm][vc] + slm_pool_f32.slm_pool_o[slmBase1 + pm * WARPSIZE * WARP_V_DIM + vg * 4 + vc] * fp32SoftMaxMul[pm]; + } + } + + [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { + if (outputMask[pm] == true) { + out_f32_vec4[offsetOutBase + outOffsets[pm] + vg] = vec4(fp32Output[pm][0], fp32Output[pm][1], fp32Output[pm][2], fp32Output[pm][3]); + } + } + } + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl index b8318c83060f..2bb444a32691 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl @@ -140,27 +140,27 @@ layout (binding = 1) readonly buffer K_PACKED_Q5_1_P32 { block_q5_1_packed32 dat FLOAT_TYPEV4 dequantize4(uint ib, uint iqs, uint a_offset, uint binding_idx) { if (binding_idx == BINDING_IDX_K) { switch (FaTypeK) { - case FA_TYPE_F32: FA_DEQUANT4_F32 (k_packed_f32) - case FA_TYPE_Q4_0: FA_DEQUANT4_Q4_0 (k_packed_q4_0) - case FA_TYPE_Q4_1: FA_DEQUANT4_Q4_1 (k_packed_q4_1) - case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0 (k_packed_q5_0) - case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1 (k_packed_q5_1) - case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0 (k_packed_q8_0) - case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL (k_packed_iq4_nl) - case FA_TYPE_BF16: FA_DEQUANT4_BF16 (k_packed_bf16) - case FA_TYPE_TURBO4_0: FA_DEQUANT4_TURBO4_0(k_packed_turbo4_0) + case GGML_TYPE_F32: FA_DEQUANT4_F32 (k_packed_f32) + case GGML_TYPE_Q4_0: FA_DEQUANT4_Q4_0(k_packed_q4_0) + case GGML_TYPE_Q4_1: FA_DEQUANT4_Q4_1(k_packed_q4_1) + case GGML_TYPE_Q5_0: FA_DEQUANT4_Q5_0(k_packed_q5_0) + case GGML_TYPE_Q5_1: FA_DEQUANT4_Q5_1(k_packed_q5_1) + case GGML_TYPE_Q8_0: FA_DEQUANT4_Q8_0(k_packed_q8_0) + case GGML_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(k_packed_iq4_nl) + case GGML_TYPE_BF16: FA_DEQUANT4_BF16(k_packed_bf16) + case GGML_TYPE_TURBO4_0: FA_DEQUANT4_TURBO4_0(k_packed_turbo4_0) } } else { switch (FaTypeV) { - case FA_TYPE_F32: FA_DEQUANT4_F32 (v_packed_f32) - case FA_TYPE_Q4_0: FA_DEQUANT4_Q4_0 (v_packed_q4_0) - case FA_TYPE_Q4_1: FA_DEQUANT4_Q4_1 (v_packed_q4_1) - case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0 (v_packed_q5_0) - case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1 (v_packed_q5_1) - case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0 (v_packed_q8_0) - case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL (v_packed_iq4_nl) - case FA_TYPE_BF16: FA_DEQUANT4_BF16 (v_packed_bf16) - case FA_TYPE_TURBO4_0: FA_DEQUANT4_TURBO4_0(v_packed_turbo4_0) + case GGML_TYPE_F32: FA_DEQUANT4_F32 (v_packed_f32) + case GGML_TYPE_Q4_0: FA_DEQUANT4_Q4_0(v_packed_q4_0) + case GGML_TYPE_Q4_1: FA_DEQUANT4_Q4_1(v_packed_q4_1) + case GGML_TYPE_Q5_0: FA_DEQUANT4_Q5_0(v_packed_q5_0) + case GGML_TYPE_Q5_1: FA_DEQUANT4_Q5_1(v_packed_q5_1) + case GGML_TYPE_Q8_0: FA_DEQUANT4_Q8_0(v_packed_q8_0) + case GGML_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(v_packed_iq4_nl) + case GGML_TYPE_BF16: FA_DEQUANT4_BF16(v_packed_bf16) + case GGML_TYPE_TURBO4_0: FA_DEQUANT4_TURBO4_0(v_packed_turbo4_0) } } return FLOAT_TYPEV4(0); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl index 6bf10a7cffd2..49900aa5aeb3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl @@ -4,20 +4,20 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { switch (FaTypeK) { - case FA_TYPE_Q4_0: { + case GGML_TYPE_Q4_0: { uint vui = pack32(u16vec2(k_packed_q4_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 0], k_packed_q4_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 1])); uint shift = (iqs & 0x10) >> 2; vui >>= shift; return int32_t(vui & 0x0F0F0F0F); } - case FA_TYPE_Q4_1: { // uses packed32 alias + case GGML_TYPE_Q4_1: { // uses packed32 alias uint vui = k_packed_q4_1_p32.data[a_offset + ib].qs[(iqs & 0xF) / 4]; uint shift = (iqs & 0x10) >> 2; vui >>= shift; return int32_t(vui & 0x0F0F0F0F); } - case FA_TYPE_Q5_0: { + case GGML_TYPE_Q5_0: { uint vui = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 0], k_packed_q5_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 1])); uint qh = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qh[0], @@ -27,7 +27,7 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { uint qh_bits = (qh >> iqs) & 0xF; return int32_t(vui & 0x0F0F0F0F) | int32_t((qh_bits * 0x02040810u) & 0x10101010u); } - case FA_TYPE_Q5_1: { // qs via packed32, qh via packed16 + case GGML_TYPE_Q5_1: { // qs via packed32, qh via packed16 uint vui = k_packed_q5_1_p32.data[a_offset + ib].qs[(iqs & 0xF) / 4]; uint qh = k_packed_q5_1.data[a_offset + ib].qh; uint shift = (iqs & 0x10) >> 2; @@ -35,7 +35,7 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { uint qh_bits = (qh >> iqs) & 0xF; return int32_t(vui & 0x0F0F0F0F) | int32_t((qh_bits * 0x02040810u) & 0x10101010u); } - case FA_TYPE_Q8_0: { + case GGML_TYPE_Q8_0: { return pack32(i16vec2(k_packed_q8_0.data[a_offset + ib].qs[iqs / 2], k_packed_q8_0.data[a_offset + ib].qs[iqs / 2 + 1])); } @@ -47,11 +47,11 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { // return (d, 0) so call sites always see the same shape. FLOAT_TYPEV2 get_k_scale(uint ib, uint a_offset) { switch (FaTypeK) { - case FA_TYPE_Q4_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + ib].d), 0.0); - case FA_TYPE_Q4_1: return FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + ib].dm); - case FA_TYPE_Q5_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + ib].d), 0.0); - case FA_TYPE_Q5_1: return FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + ib].dm); - case FA_TYPE_Q8_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + ib].d), 0.0); + case GGML_TYPE_Q4_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + ib].d), 0.0); + case GGML_TYPE_Q4_1: return FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + ib].dm); + case GGML_TYPE_Q5_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + ib].d), 0.0); + case GGML_TYPE_Q5_1: return FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + ib].dm); + case GGML_TYPE_Q8_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + ib].d), 0.0); default: return FLOAT_TYPEV2(0); } } @@ -61,16 +61,16 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, c // explicit casts. The bit pattern is what we care about here -- the actual // signed/unsigned interpretation happens downstream in the dot product. switch (FaTypeK) { - case FA_TYPE_Q4_0: { + case GGML_TYPE_Q4_0: { kblocksh[buf_ib].qs[iqs] = int32_t(pack32(u16vec2(k_packed_q4_0.data[a_offset + global_ib].qs[iqs * 2], k_packed_q4_0.data[a_offset + global_ib].qs[iqs * 2 + 1]))); break; } - case FA_TYPE_Q4_1: { + case GGML_TYPE_Q4_1: { kblocksh[buf_ib].qs[iqs] = int32_t(k_packed_q4_1_p32.data[a_offset + global_ib].qs[iqs]); break; } - case FA_TYPE_Q5_0: { + case GGML_TYPE_Q5_0: { kblocksh[buf_ib].qs[iqs] = int32_t(pack32(u16vec2(k_packed_q5_0.data[a_offset + global_ib].qs[iqs * 2], k_packed_q5_0.data[a_offset + global_ib].qs[iqs * 2 + 1]))); if (iqs == 0) { @@ -79,14 +79,14 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, c } break; } - case FA_TYPE_Q5_1: { + case GGML_TYPE_Q5_1: { kblocksh[buf_ib].qs[iqs] = int32_t(k_packed_q5_1_p32.data[a_offset + global_ib].qs[iqs]); if (iqs == 0) { kblocksh[buf_ib].qh = k_packed_q5_1.data[a_offset + global_ib].qh; } break; } - case FA_TYPE_Q8_0: { + case GGML_TYPE_Q8_0: { kblocksh[buf_ib].qs[iqs] = pack32(i16vec2(k_packed_q8_0.data[a_offset + global_ib].qs[iqs * 2], k_packed_q8_0.data[a_offset + global_ib].qs[iqs * 2 + 1])); break; @@ -96,11 +96,11 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, c if (iqs == 0) { // Q4_0/Q5_0/Q8_0 store dm.x = d; Q4_1/Q5_1 store dm = (d, m) pair. switch (FaTypeK) { - case FA_TYPE_Q4_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + global_ib].d), 0.0); break; - case FA_TYPE_Q4_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + global_ib].dm); break; - case FA_TYPE_Q5_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + global_ib].d), 0.0); break; - case FA_TYPE_Q5_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + global_ib].dm); break; - case FA_TYPE_Q8_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + global_ib].d), 0.0); break; + case GGML_TYPE_Q4_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + global_ib].d), 0.0); break; + case GGML_TYPE_Q4_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + global_ib].dm); break; + case GGML_TYPE_Q5_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + global_ib].d), 0.0); break; + case GGML_TYPE_Q5_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + global_ib].dm); break; + case GGML_TYPE_Q8_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + global_ib].d), 0.0); break; } } } @@ -121,31 +121,31 @@ struct fa_k_qs_block8 { fa_k_qs_block8 get_k_qs_block8(uint ib, uint a_offset) { fa_k_qs_block8 r; uint qh = 0; - if (FaTypeK == FA_TYPE_Q5_0) { + if (FaTypeK == GGML_TYPE_Q5_0) { qh = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qh[0], k_packed_q5_0.data[a_offset + ib].qh[1])); - } else if (FaTypeK == FA_TYPE_Q5_1) { + } else if (FaTypeK == GGML_TYPE_Q5_1) { qh = k_packed_q5_1.data[a_offset + ib].qh; } - const bool has_qh = (FaTypeK == FA_TYPE_Q5_0) || (FaTypeK == FA_TYPE_Q5_1); + const bool has_qh = (FaTypeK == GGML_TYPE_Q5_0) || (FaTypeK == GGML_TYPE_Q5_1); [[unroll]] for (uint32_t d = 0; d < 4; d++) { uint vui = 0; switch (FaTypeK) { - case FA_TYPE_Q4_0: { // packed16 + case GGML_TYPE_Q4_0: { // packed16 vui = pack32(u16vec2(k_packed_q4_0.data[a_offset + ib].qs[d * 2 + 0], k_packed_q4_0.data[a_offset + ib].qs[d * 2 + 1])); break; } - case FA_TYPE_Q4_1: { // packed32 alias + case GGML_TYPE_Q4_1: { // packed32 alias vui = k_packed_q4_1_p32.data[a_offset + ib].qs[d]; break; } - case FA_TYPE_Q5_0: { // packed16 + case GGML_TYPE_Q5_0: { // packed16 vui = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qs[d * 2 + 0], k_packed_q5_0.data[a_offset + ib].qs[d * 2 + 1])); break; } - case FA_TYPE_Q5_1: { // packed32 alias + case GGML_TYPE_Q5_1: { // packed32 alias vui = k_packed_q5_1_p32.data[a_offset + ib].qs[d]; break; } @@ -164,21 +164,21 @@ fa_k_qs_block8 get_k_qs_block8(uint ib, uint a_offset) { int32_t get_k_qs_shmem(const uint buf_ib, const uint pos) { switch (FaTypeK) { - case FA_TYPE_Q4_0: - case FA_TYPE_Q4_1: { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: { uint sub = pos % 4; uint shift = ((pos % 8) >= 4) ? 4u : 0u; return int32_t((uint(kblocksh[buf_ib].qs[sub]) >> shift) & 0x0F0F0F0Fu); } - case FA_TYPE_Q5_0: - case FA_TYPE_Q5_1: { + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: { uint sub = pos % 4; uint shift = ((pos % 8) >= 4) ? 4u : 0u; int32_t result = int32_t((uint(kblocksh[buf_ib].qs[sub]) >> shift) & 0x0F0F0F0Fu); uint qh_bits = (kblocksh[buf_ib].qh >> (pos * 4u)) & 0xFu; return result | int32_t((qh_bits * 0x02040810u) & 0x10101010u); } - case FA_TYPE_Q8_0: { + case GGML_TYPE_Q8_0: { return kblocksh[buf_ib].qs[pos]; } default: return 0; @@ -187,10 +187,10 @@ int32_t get_k_qs_shmem(const uint buf_ib, const uint pos) { ACC_TYPE k_dot_correction(const uint qib, const ACC_TYPEV2 k_dm) { switch (FaTypeK) { - case FA_TYPE_Q4_0: return -ACC_TYPE(8.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; - case FA_TYPE_Q5_0: return -ACC_TYPE(16.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; - case FA_TYPE_Q4_1: - case FA_TYPE_Q5_1: return ACC_TYPE(Qf[qib].ds.y) * k_dm.y; + case GGML_TYPE_Q4_0: return -ACC_TYPE(8.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; + case GGML_TYPE_Q5_0: return -ACC_TYPE(16.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_1: return ACC_TYPE(Qf[qib].ds.y) * k_dm.y; default: return ACC_TYPE(0.0); } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_sparse_compact.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_sparse_compact.comp new file mode 100644 index 000000000000..3d31362661b3 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_sparse_compact.comp @@ -0,0 +1,102 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : enable +#extension GL_EXT_shader_16bit_storage : require +#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require +#ifdef USE_SUBGROUPS +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_ballot : require +#endif + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +layout(constant_id = 0) const uint BLOCK_SIZE = 128; +layout(constant_id = 1) const uint NUM_SUBGROUPS = 1; + +layout (binding = 0) readonly buffer M {float16_t data_m[];}; +layout (binding = 1) writeonly buffer I {int32_t data_i[];}; + +layout (push_constant) uniform parameter { + uint KV; + uint nem1; + uint nem2; + uint nbm1; + uint nbm2; + uint nbm3; + uint n_kv_max; +} p; + +#ifdef USE_SUBGROUPS +shared uvec4 ballots_sh[NUM_SUBGROUPS]; +#else +shared uint scan[BLOCK_SIZE]; +#endif + +// One workgroup per mask row: compact the finite-mask KV positions into a +// per-row index list of length n_kv_max, -1 padded. Emitted in ascending KV +// order so the downstream attention accumulation is deterministic. +void main() { + const uint i1 = gl_WorkGroupID.x; + const uint i2 = gl_WorkGroupID.y; + const uint i3 = gl_WorkGroupID.z; + const uint tid = gl_LocalInvocationIndex; + + const uint m_base = i3 * p.nbm3 + i2 * p.nbm2 + i1 * p.nbm1; + const uint out_base = ((i3 * p.nem2 + i2) * p.nem1 + i1) * p.n_kv_max; + + uint base = 0; + for (uint chunk = 0; chunk < p.KV; chunk += BLOCK_SIZE) { + const uint k = chunk + tid; + bool selected = false; + if (k < p.KV) { + const float v = float(data_m[m_base + k]); + selected = !isinf(v) && !isnan(v); + } + +#ifdef USE_SUBGROUPS + const uvec4 ballot = subgroupBallot(selected); + if (subgroupElect()) { + ballots_sh[gl_SubgroupID] = ballot; + } + barrier(); + + uint subgroup_base = 0; + uint total = 0; + [[unroll]] for (uint s = 0; s < gl_NumSubgroups; ++s) { + if (s == gl_SubgroupID) { + subgroup_base = total; + } + total += subgroupBallotBitCount(ballots_sh[s]); + } + barrier(); + + const uint slot = base + subgroup_base + subgroupBallotExclusiveBitCount(ballot); +#else + // Hillis-Steele inclusive prefix sum over the workgroup. + scan[tid] = selected ? 1u : 0u; + barrier(); + for (uint off = 1; off < BLOCK_SIZE; off <<= 1) { + uint add = 0; + if (tid >= off) { + add = scan[tid - off]; + } + barrier(); + scan[tid] += add; + barrier(); + } + + const uint inclusive = scan[tid]; + const uint total = scan[BLOCK_SIZE - 1]; + const uint slot = base + inclusive - 1u; +#endif + + if (selected && slot < p.n_kv_max) { + data_i[out_base + slot] = int32_t(k); + } + base += total; + barrier(); + } + + for (uint s = min(base, p.n_kv_max) + tid; s < p.n_kv_max; s += BLOCK_SIZE) { + data_i[out_base + s] = int32_t(-1); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp b/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp index 9dba437edbee..19af30ac98fa 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp @@ -27,10 +27,10 @@ void main() { const uint i11 = gid_z / p.ne12; const uint i12 = gid_z % p.ne12; - const uint i01 = data_b[i10*p.nb10 + i11*p.nb11 + i12*p.nb12]; + const uint i01 = data_b[get_boffset() + i10*p.nb10 + i11*p.nb11 + i12*p.nb12]; - const uint a_offset = i01*p.nb01 + i11*p.nb02 + i12*p.nb03; - const uint d_offset = i10*p.nb21 + i11*p.nb22 + i12*p.nb23; + const uint a_offset = get_aoffset() + i01*p.nb01 + i11*p.nb02 + i12*p.nb03; + const uint d_offset = get_doffset() + i10*p.nb21 + i11*p.nb22 + i12*p.nb23; const uint ib = a_offset + i00/QUANT_K; // block index const uint iqs = (i00%QUANT_K)/QUANT_R; // quant index diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl new file mode 100644 index 000000000000..65994ca2c6e3 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl @@ -0,0 +1,38 @@ +#if !defined(GGML_TYPE_IDS_COMP) +#define GGML_TYPE_IDS_COMP + +// ggml_type enum values — must match ggml.h +#define GGML_TYPE_F32 0u +#define GGML_TYPE_F16 1u +#define GGML_TYPE_Q4_0 2u +#define GGML_TYPE_Q4_1 3u +#define GGML_TYPE_Q5_0 6u +#define GGML_TYPE_Q5_1 7u +#define GGML_TYPE_Q8_0 8u +#define GGML_TYPE_Q2_K 10u +#define GGML_TYPE_Q3_K 11u +#define GGML_TYPE_Q4_K 12u +#define GGML_TYPE_Q5_K 13u +#define GGML_TYPE_Q6_K 14u +#define GGML_TYPE_IQ2_XXS 16u +#define GGML_TYPE_IQ2_XS 17u +#define GGML_TYPE_IQ3_XXS 18u +#define GGML_TYPE_IQ1_S 19u +#define GGML_TYPE_IQ4_NL 20u +#define GGML_TYPE_IQ3_S 21u +#define GGML_TYPE_IQ2_S 22u +#define GGML_TYPE_IQ4_XS 23u +#define GGML_TYPE_IQ1_M 29u +#define GGML_TYPE_BF16 30u +#define GGML_TYPE_TQ1_0 34u +#define GGML_TYPE_TQ2_0 35u +#define GGML_TYPE_MXFP4 39u +#define GGML_TYPE_NVFP4 40u +#define GGML_TYPE_Q1_0 41u +// fork: 42..55 are TurboQuant/ml8 types here, so Q2_0 moved to 56 (see ggml.h) +#define GGML_TYPE_TURBO3_0 42u +#define GGML_TYPE_TURBO4_0 43u +#define GGML_TYPE_TURBO2_0 44u +#define GGML_TYPE_Q2_0 56u + +#endif // !defined(GGML_TYPE_IDS_COMP) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp b/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp index f4130d223b13..ea77a3d7a036 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp @@ -31,7 +31,7 @@ layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; #if BDA -layout (buffer_reference) buffer D_ptr {D_TYPE d;}; +layout (buffer_reference, buffer_reference_align = D_SIZE) buffer D_ptr {D_TYPE d;}; #endif void im2col(const uint ow, const uint z_idx) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp b/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp index 93f61fd85435..64ae7e4fd612 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp @@ -50,7 +50,7 @@ layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; #if BDA -layout (buffer_reference) buffer D_ptr {D_TYPE d;}; +layout (buffer_reference, buffer_reference_align = D_SIZE) buffer D_ptr {D_TYPE d;}; #endif void main() { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl new file mode 100644 index 000000000000..12e50ee9eb9d --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl @@ -0,0 +1,2 @@ +void init_iq_shmem(uvec3 wgsize) { +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp b/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp index ba76ec72ca6c..9b34d8366220 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp @@ -12,9 +12,9 @@ #include "types.glsl" #include "fa_types.glsl" -#define FaTypeV FA_TYPE_F32 +#define FaTypeV GGML_TYPE_F32 -layout(constant_id = 0) const uint FaTypeK = FA_TYPE_F32; +layout(constant_id = 0) const uint FaTypeK = GGML_TYPE_F32; layout(constant_id = 1) const uint FaBlockBytesK = 4; layout(constant_id = 2) const uint SUBGROUP_SIZE = 32; @@ -84,11 +84,11 @@ void main() { const uint k_block_elems = fa_block_elems(FaTypeK); const uint k_elem_bytes = FaBlockBytesK / k_block_elems; - if (FaTypeK == FA_TYPE_F16) { + if (FaTypeK == GGML_TYPE_F16) { k_row[tid] = float(k_f16[k_offset / k_elem_bytes + tid]); - } else if (FaTypeK == FA_TYPE_F32) { + } else if (FaTypeK == GGML_TYPE_F32) { k_row[tid] = k_f32[k_offset / k_elem_bytes + tid]; - } else if (FaTypeK == FA_TYPE_BF16) { + } else if (FaTypeK == GGML_TYPE_BF16) { k_row[tid] = bf16_to_fp32(uint(k_bf16[k_offset / k_elem_bytes + tid])); } else if (4 * tid < HEAD_SIZE) { const uint coord = 4 * tid; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq4_xs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq4_xs.comp new file mode 100644 index 000000000000..a2b99d9ab16c --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq4_xs.comp @@ -0,0 +1,97 @@ +#version 450 + +#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require + +#include "mul_mat_vec_base.glsl" + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; + +// dedicated iq4_xs mat-vec, mirrors mul_mat_vec_iq3_s.comp +// one packed32 word per l, so the 6-bit subblock scale is hoisted to a single fma after register accumulation + +void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 32 * ib32; + + uint ibi = a_offset + first_row * num_blocks_per_row + i; + [[unroll]] for (uint n = 0; n < num_rows; ++n) { + const float d = float(data_a[ibi].d); + const uint sl = (data_a[ibi].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; + const uint sh = (data_a[ibi].scales_h >> (2 * ib32)) & 3; + const float dscale = d * float(int(sl | (sh << 4)) - 32); + + FLOAT_TYPE sum[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + sum[j] = FLOAT_TYPE(0); + } + + [[unroll]] for (uint l = 0; l < 4; ++l) { + const uint w = data_a_packed32[ibi].qs[4 * ib32 + l]; + const u8vec4 q0 = unpack8(w & 0x0F0F0F0F); + const u8vec4 q1 = unpack8((w >> 4) & 0x0F0F0F0F); + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + l]); + const vec4 b1 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 4 + l]); + + sum[j] = fma(FLOAT_TYPE(b0.x), FLOAT_TYPE(kvalues_iq4nl[q0.x]), + fma(FLOAT_TYPE(b0.y), FLOAT_TYPE(kvalues_iq4nl[q0.y]), + fma(FLOAT_TYPE(b0.z), FLOAT_TYPE(kvalues_iq4nl[q0.z]), + fma(FLOAT_TYPE(b0.w), FLOAT_TYPE(kvalues_iq4nl[q0.w]), + fma(FLOAT_TYPE(b1.x), FLOAT_TYPE(kvalues_iq4nl[q1.x]), + fma(FLOAT_TYPE(b1.y), FLOAT_TYPE(kvalues_iq4nl[q1.y]), + fma(FLOAT_TYPE(b1.z), FLOAT_TYPE(kvalues_iq4nl[q1.z]), + fma(FLOAT_TYPE(b1.w), FLOAT_TYPE(kvalues_iq4nl[q1.w]), + sum[j])))))))); + } + } + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + temp[j][n] = fma(dscale, sum[j], temp[j][n]); + } + + ibi += num_blocks_per_row; + } +} + +void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { + uint a_offset, b_offset, d_offset; + + get_offsets(a_offset, b_offset, d_offset); + + const uint num_blocks_per_row = p.ncols / QUANT_K; + + // 8 threads are used to process each block + const uint blocks_per_wg = gl_WorkGroupSize.x/8; + const uint tid = gl_LocalInvocationID.x; + const uint itid = tid % 8; // 0...7 + const uint ix = tid / 8; + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) { + temp[j][i] = FLOAT_TYPE(0); + } + } + + [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg) + calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows); + + reduce_result(temp, d_offset, first_row, num_rows, tid); +} + +void main() { + const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); + + init_iq_shmem(gl_WorkGroupSize); + + // do NUM_ROWS at a time, unless there aren't enough remaining rows + if (first_row + NUM_ROWS <= p.stride_d) { + compute_outputs(first_row, NUM_ROWS); + } else { + if (first_row >= p.stride_d) { + return; + } + compute_outputs(first_row, p.stride_d - first_row); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq1_0.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq1_0.comp new file mode 100644 index 000000000000..2c99a268e6da --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq1_0.comp @@ -0,0 +1,85 @@ +#version 450 +#extension GL_EXT_shader_explicit_arithmetic_types : require + +#include "mul_mat_vec_base.glsl" + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; + +// Walks the packed bytes directly (byte m, digit t) rather than via +// tq1_0_byte_of()/tq1_0_digit_of(): one byte per thread, expanded in place. +void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { + uint a_offset, b_offset, d_offset; + get_offsets(a_offset, b_offset, d_offset); + + const uint num_blocks_per_row = p.ncols / QUANT_K; + const uint tid = gl_LocalInvocationID.x; + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) { + temp[j][i] = FLOAT_TYPE(0); + } + } + + for (uint nrow = 0; nrow < num_rows; ++nrow) { + const uint ib0 = a_offset + (first_row + nrow) * num_blocks_per_row; + for (uint jcol = 0; jcol < NUM_COLS; ++jcol) { + const uint b_base = (jcol * p.batch_stride_b); + for (uint i = tid/8; i < num_blocks_per_row; i += gl_WorkGroupSize.x/8) { + const FLOAT_TYPE d = float(data_a[ib0 + i].d); + + // First qs chunk: 32 bytes (5*32 elements) + [[unroll]] for (uint m = tid%8; m < 32; m += 8) { + const uint q_byte = uint(data_a[ib0 + i].qs[m]); + [[unroll]] for (uint t = 0; t < 5; ++t) { + const uint xi = tq1_0_trit(q_byte, t); + const FLOAT_TYPE dequant_val = FLOAT_TYPE(d * (float(xi) - 1.0f)); + const uint elem = t * 32u + m; + const uint b_idx = i * QUANT_K + elem; + temp[jcol][nrow] += dequant_val * FLOAT_TYPE(data_b[b_base + b_offset + b_idx]); + } + } + + // Second qs chunk: 16 bytes (5*16 elements) + [[unroll]] for (uint m = tid%8; m < 16; m += 8) { + const uint q_byte = uint(data_a[ib0 + i].qs[32u + m]); + [[unroll]] for (uint t = 0; t < 5; ++t) { + const uint xi = tq1_0_trit(q_byte, t); + const FLOAT_TYPE dequant_val = FLOAT_TYPE(d * (float(xi) - 1.0f)); + const uint elem = 160u + t * 16u + m; + const uint b_idx = i * QUANT_K + elem; + temp[jcol][nrow] += dequant_val * FLOAT_TYPE(data_b[b_base + b_offset + b_idx]); + } + } + + // qh bytes: 4 bytes (4*4 elements) + [[unroll]] for (uint j = tid%8; j < 4; j += 8) { + const uint qh_byte = uint(data_a[ib0 + i].qh[j]); + [[unroll]] for (uint t = 0; t < 4; ++t) { + const uint xi = tq1_0_trit(qh_byte, t); + const FLOAT_TYPE dequant_val = FLOAT_TYPE(d * (float(xi) - 1.0f)); + const uint elem = 240u + t * 4u + j; + const uint b_idx = i * QUANT_K + elem; + temp[jcol][nrow] += dequant_val * FLOAT_TYPE(data_b[b_base + b_offset + b_idx]); + } + } + } + } + } + + reduce_result(temp, d_offset, first_row, num_rows, tid); +} + +void main() { + const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); + + if (first_row + NUM_ROWS <= p.stride_d) { + compute_outputs(first_row, NUM_ROWS); + } else { + if (first_row >= p.stride_d) { + return; + } + compute_outputs(first_row, p.stride_d - first_row); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp index 18d441ead40e..3383aa88ef91 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp @@ -5,6 +5,7 @@ #define MMQ #define NEEDS_IQ1S_GRID_GPU +#define KVALUES_IQ4NL_I8 #define B_TYPE block_q8_1_x4 #include "mul_mat_vec_base.glsl" @@ -15,7 +16,7 @@ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; #define K_PER_ITER 16 #elif defined(DATA_A_QUANT_LEGACY) || defined(DATA_A_MXFP4) #define K_PER_ITER 8 -#elif defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M) +#elif defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M) || defined(DATA_A_IQ4_XS) #define K_PER_ITER 32 #else #error unimplemented diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl index a5403ac82121..03c348bf9e3c 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl @@ -448,6 +448,28 @@ FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) { } #endif +#if defined(DATA_A_IQ4_XS) +FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) { + const uint ib = ib_a / 8; + const uint ib32 = ib_a % 8; + + int32_t q_sum = 0; + [[unroll]] for (uint j = 0; j < 4; ++j) { + const uint32_t vui = data_a_packed32[ib].qs[4 * ib32 + j]; + const i32vec2 qs_a = iq4nl_to_i8x8(vui); + + q_sum += dotPacked4x8EXT(qs_a.x, cache_b_qs[j]); + q_sum += dotPacked4x8EXT(qs_a.y, cache_b_qs[j + 4]); + } + + const uint sl = (data_a_packed32[ib].scales_l >> (4 * ib32)) & 0xF; + const uint sh = (data_a_packed32[ib].scales_h >> (2 * ib32)) & 3; + const float d = float(data_a[ib].d) * float(int(sl | (sh << 4)) - 32); + + return FLOAT_TYPE(float(cache_b_ds.x) * d * float(q_sum)); +} +#endif + #if defined(DATA_A_IQ1_S) void repack8(uint ib, uint iqs, out i32vec4 out0, out i32vec4 out1) { const uint ib32 = iqs / 32; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index 76553168e3f7..b8ea15b52470 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -9,6 +9,9 @@ #if defined(DATA_A_IQ1_M) #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #endif +#if !defined(DATA_A_F32) && !defined(DATA_A_F16) && !defined(DATA_A_BF16) +#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require +#endif #if defined(DATA_A_BF16) && defined(COOPMAT) #extension GL_EXT_bfloat16 : enable @@ -28,24 +31,54 @@ #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #endif +#ifdef MULMAT_QUANT +#include "ggml_type_ids.glsl" +layout (constant_id = 12) const uint MmTypeA = 0; +#endif + #include "types.glsl" #include "dot_product_funcs.glsl" +#ifndef MULMAT_QUANT #ifndef LOAD_VEC_A #define LOAD_VEC_A 1 #endif +#endif #ifndef LOAD_VEC_B #define LOAD_VEC_B 1 #endif layout (constant_id = 11) const uint ALIGNED = 0; +#ifdef MULMAT_QUANT + +uint mm_load_vec_a() { + switch (MmTypeA) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_1: + return 8u; + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return 4u; + default: + return 2u; + } +} +#endif + #if !defined(TO_FLOAT_TYPE) #define TO_FLOAT_TYPE FLOAT_TYPE #endif layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +#ifndef MULMAT_QUANT layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; #if defined(DATA_A_F32) layout (binding = 0) readonly buffer A_SCALAR {float data_a_scalar[];}; @@ -60,6 +93,30 @@ layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16 #if defined(A_TYPE_PACKED32) layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];}; #endif +#else +// Unpacked struct aliases +layout (binding = 0) readonly buffer BUF_Q1_0 { block_q1_0 data[]; } a_q1_0; +layout (binding = 0) readonly buffer BUF_Q2_0 { block_q2_0 data[]; } a_q2_0; +layout (binding = 0) readonly buffer BUF_Q2_K { block_q2_K data[]; } a_q2_k; +layout (binding = 0) readonly buffer BUF_Q3_K { block_q3_K data[]; } a_q3_k; +layout (binding = 0) readonly buffer BUF_Q4_K { block_q4_K data[]; } a_q4_k; +layout (binding = 0) readonly buffer BUF_Q5_K { block_q5_K data[]; } a_q5_k; +layout (binding = 0) readonly buffer BUF_Q6_K { block_q6_K data[]; } a_q6_k; +layout (binding = 0) readonly buffer BUF_TQ1_0 { block_tq1_0 data[]; } a_tq1_0; +layout (binding = 0) readonly buffer BUF_TQ2_0 { block_tq2_0 data[]; } a_tq2_0; +// Packed16 aliases +layout (binding = 0) readonly buffer BUF_Q4_0_P16 { block_q4_0_packed16 data[]; } a_q4_0_p16; +layout (binding = 0) readonly buffer BUF_Q5_0_P16 { block_q5_0_packed16 data[]; } a_q5_0_p16; +layout (binding = 0) readonly buffer BUF_Q8_0_P16 { block_q8_0_packed16 data[]; } a_q8_0_p16; +layout (binding = 0) readonly buffer BUF_Q3_K_P16 { block_q3_K_packed16 data[]; } a_q3_k_p16; +layout (binding = 0) readonly buffer BUF_Q6_K_P16 { block_q6_K_packed16 data[]; } a_q6_k_p16; +// Packed32 aliases +layout (binding = 0) readonly buffer BUF_Q4_1_P32 { block_q4_1_packed32 data[]; } a_q4_1_p32; +layout (binding = 0) readonly buffer BUF_Q5_1_P32 { block_q5_1_packed32 data[]; } a_q5_1_p32; +layout (binding = 0) readonly buffer BUF_Q2_K_P32 { block_q2_K_packed32 data[]; } a_q2_k_p32; +layout (binding = 0) readonly buffer BUF_Q4_K_P32 { block_q4_K_packed32 data[]; } a_q4_k_p32; +layout (binding = 0) readonly buffer BUF_Q5_K_P32 { block_q5_K_packed32 data[]; } a_q5_k_p32; +#endif layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 1) readonly buffer B_SCALAR {B_TYPE_SCALAR data_b_scalar[];}; @@ -129,8 +186,13 @@ layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working wit #endif #ifdef COOPMAT +#ifdef MULMAT_QUANT +layout(constant_id = 13) const uint SHMEM_STRIDE_PAD = 4; +layout(constant_id = 14) const bool APPLY_SLM_A_RESHAPE = false; +#else layout(constant_id = 12) const uint SHMEM_STRIDE_PAD = 4; layout(constant_id = 13) const bool APPLY_SLM_A_RESHAPE = false; +#endif #else const uint SHMEM_STRIDE_PAD = 1; const bool APPLY_SLM_A_RESHAPE = false; @@ -149,6 +211,10 @@ shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS]; #include "mul_mm_id_funcs.glsl" #include "mul_mm_funcs.glsl" +#ifdef MULMAT_QUANT +#include "iq_shmem_init.glsl" +#endif + void main() { const uint ic = gl_WorkGroupID.y; @@ -158,7 +224,7 @@ void main() { return; } #endif -#ifdef NEEDS_INIT_IQ_SHMEM +#if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT) init_iq_shmem(gl_WorkGroupSize); #endif @@ -208,9 +274,12 @@ void main() { #if defined(DATA_A_F32) || defined(DATA_A_F16) || defined(DATA_A_BF16) const uint LOAD_VEC_A_EFF = (ALIGNED != 0) ? LOAD_VEC_A : 1; const uint LOAD_VEC_BATCH_A = (ALIGNED != 0) ? 1 : 2; -#else +#elif !defined(MULMAT_QUANT) const uint LOAD_VEC_A_EFF = LOAD_VEC_A; const uint LOAD_VEC_BATCH_A = 1; +#else + const uint LOAD_VEC_A_EFF = mm_load_vec_a(); + const uint LOAD_VEC_BATCH_A = 1; #endif const uint LOAD_VEC_B_EFF = (ALIGNED != 0) ? LOAD_VEC_B : 1; const uint LOAD_VEC_BATCH_B = (ALIGNED != 0) ? 1 : 2; @@ -252,6 +321,9 @@ void main() { // Workgroup has no work if (ic * BN >= _ne1) return; + + uint required_work_items = (_ne1 - ic * BN) * BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B; + uint required_warp_c = (_ne1 - ic * BN + WN - 1) / WN; #endif #ifdef MUL_MAT_ID @@ -305,6 +377,9 @@ void main() { [[unroll]] for (uint l = 0; l < BM; l += loadstride_a) { load_a_to_shmem(pos_a, loadr_a, loadc_a + l, ir * BM + loadc_a + l, block, end_k); } + #ifdef MUL_MAT_ID + if (gl_LocalInvocationID.x < required_work_items) { + #endif [[unroll]] for (uint l = 0; l < BN; l += loadstride_b) { #if !defined(MUL_MAT_ID) load_b_to_shmem(pos_b, loadr_b, loadc_b + l, ic * BN + loadc_b + l, block, end_k); @@ -312,6 +387,9 @@ void main() { load_b_to_shmem(pos_b, loadr_b, loadc_b + l, ic, _ne1, block, end_k); #endif } + #ifdef MUL_MAT_ID + } + #endif barrier(); @@ -319,6 +397,9 @@ void main() { pos_b += BK / LOAD_VEC_B_EFF; #ifdef COOPMAT +#ifdef MUL_MAT_ID + if (warp_c < required_warp_c) { +#endif [[unroll]] for (uint i = 0; i < BK; i += TK) { [[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) { // Load from shared into cache @@ -331,6 +412,9 @@ void main() { } } } +#ifdef MUL_MAT_ID + } +#endif #else [[unroll]] for (uint i = 0; i < BK / BK_STEP; i++) { // Load from shared into cache diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 27f3178e7f26..9b59e8bca799 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -21,6 +21,13 @@ #extension GL_EXT_bfloat16 : enable #endif +#include "ggml_type_ids.glsl" + +#ifdef MULMAT_QUANT +layout (constant_id = 7) const uint MmTypeA = 0; +layout (constant_id = 8) const uint MmABlockBytes = 2; +#endif + #include "types.glsl" #include "utils.glsl" @@ -34,9 +41,27 @@ layout (constant_id = 2) const uint BN = 64; layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working with a quant layout (constant_id = 4) const bool enable_smaller_matrices = false; -const uint BNover2 = enable_smaller_matrices ? (BN / 2) : BN; -const uint BNover4 = enable_smaller_matrices ? (BN / 4) : BN; +const uint BNover2 = BN / 2; +const uint BNover4 = enable_smaller_matrices ? (BN / 4) : (BN / 2); layout (constant_id = 5) const uint ALIGNED = 0; +layout (constant_id = 6) const uint subgroup_size = 32; + +#ifdef MULMAT_QUANT + +uint mm_quant_k() { + switch (MmTypeA) { + case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return 32u; + case GGML_TYPE_Q1_0: + return 128u; + case GGML_TYPE_Q2_0: + return 64u; + default: + return 256u; + } +} +#endif layout (push_constant) uniform parameter { @@ -72,21 +97,73 @@ layout (push_constant) uniform parameter } p; +#ifndef MULMAT_QUANT layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; +#else +layout (binding = 0) readonly buffer A {uint8_t data_a[];}; +#endif layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; #if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR) layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];}; #endif -#if QUANT_K > 1 +#if defined(MULMAT_QUANT) || QUANT_K > 1 #include "dequant_funcs_cm2.glsl" +#ifndef MULMAT_QUANT +// Per-type path: use the alias set by dequant_funcs_cm2.glsl #if defined(dequantFuncA_v) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR) #define DECODEFUNCA , dequantFuncA, dequantFuncA_v #else #define DECODEFUNCA , dequantFuncA #endif #else +layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufA { + uint8_t raw[MmABlockBytes]; +}; + +float16_t mmDecodeA(const in decodeBufA bl_in, const in uint blockCoords[2], const in uint coordInBlock[2]) { + switch (MmTypeA) { + case GGML_TYPE_Q1_0: return dequantFuncQ1_0 (decodeBufQ1_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_0: return dequantFuncQ2_0 (decodeBufQ2_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0 (decodeBufQ4_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1 (decodeBufQ4_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0 (decodeBufQ5_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1 (decodeBufQ5_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0 (decodeBufQ8_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_K: return dequantFuncQ2_K (decodeBufQ2_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q3_K: return dequantFuncQ3_K (decodeBufQ3_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q6_K: return dequantFuncQ6_K (decodeBufQ6_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_TQ1_0: return dequantFuncTQ1_0(decodeBufTQ1_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_TQ2_0: return dequantFuncTQ2_0(decodeBufTQ2_0(bl_in), blockCoords, coordInBlock); + default: return float16_t(0); + } +} + +#ifdef GGML_VULKAN_COOPMAT2_DECODE_VECTOR +f16vec4 mmDecodeA_v(const in decodeBufA bl_in, const in uint blockCoords[2], const in uint coordInBlock[2]) { + switch (MmTypeA) { + case GGML_TYPE_Q1_0: return dequantFuncQ1_0_v (decodeBufQ1_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_0: return dequantFuncQ2_0_v (decodeBufQ2_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v (decodeBufQ4_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v (decodeBufQ4_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v (decodeBufQ5_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v (decodeBufQ5_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v (decodeBufQ8_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_K: return dequantFuncQ2_K_v (decodeBufQ2_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q3_K: return dequantFuncQ3_K_v (decodeBufQ3_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q6_K: return dequantFuncQ6_K_v (decodeBufQ6_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_TQ1_0: return dequantFuncTQ1_0_v(decodeBufTQ1_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_TQ2_0: return dequantFuncTQ2_0_v(decodeBufTQ2_0(bl_in), blockCoords, coordInBlock); + default: return f16vec4(0); + } +} +#define DECODEFUNCA , mmDecodeA, mmDecodeA_v +#else +#define DECODEFUNCA , mmDecodeA +#endif +#endif +#else #define DECODEFUNCA #endif @@ -114,7 +191,6 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufB { }; uint _ne1; -layout (constant_id = 6) const uint subgroup_size = 32; shared uvec4 ballots_sh[BLOCK_SIZE / subgroup_size]; B_TYPE decodeFuncB(const in decodeBufB bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -246,6 +322,10 @@ void load_row_ids_hoisted(uint expert_idx, uint ic) { } #endif +#ifdef MULMAT_QUANT +#include "iq_shmem_init.glsl" +#endif + void main() { const uint tid = gl_LocalInvocationIndex; const uint ic = gl_WorkGroupID.y; @@ -264,7 +344,7 @@ void main() { #endif #endif -#ifdef NEEDS_INIT_IQ_SHMEM +#if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT) init_iq_shmem(gl_WorkGroupSize); #endif @@ -305,22 +385,33 @@ void main() { const uint end_k = min(p.K, (ik + 1) * p.k_split); #endif +#ifdef MULMAT_QUANT + const uint qk = mm_quant_k(); +#else + const uint qk = QUANT_K; +#endif + #ifdef MUL_MAT_ID - uint pos_a = expert_idx * (p.batch_stride_a / QUANT_K); + uint pos_a = expert_idx * (p.batch_stride_a / qk); uint pos_b = 0; #else - uint pos_a = batch_idx_a * (p.batch_stride_a / QUANT_K); + uint pos_a = batch_idx_a * (p.batch_stride_a / qk); uint pos_b = batch_idx * p.batch_stride_b; uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches; #endif - uint stride_a = p.stride_a / QUANT_K; +#ifdef MULMAT_QUANT + // pos_a is a byte offset into the raw buffer; strides stay in block units + pos_a *= MmABlockBytes; +#endif + + uint stride_a = p.stride_a / qk; uint stride_b = p.stride_b; // Hint to the compiler that values are aligned (want 16B alignment). // Quants are always block-aligned, no alignment needed. if (ALIGNED != 0) { -#if QUANT_K == 1 +#if !defined(MULMAT_QUANT) && QUANT_K == 1 stride_a &= ~7; #endif stride_b &= ~7; @@ -335,10 +426,8 @@ void main() { #endif tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutD = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV); -#if QUANT_K > 1 - tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K); - tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, QUANT_K); -#endif + tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, qk); + tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, qk); #if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR) tensorLayoutB = setTensorLayoutBlockSizeNV(tensorLayoutB, 1, BK); #endif @@ -368,19 +457,19 @@ void main() { const uint START_ALIGN_K = 256; // For Qi_K (block size 256), unroll whole 256 element tiles. // For legacy quants (block size 32), unroll 8x. - const uint UNROLL_K = (QUANT_K == 256) ? 256 : (BK * 8); + const uint UNROLL_K = (qk == 256) ? 256 : (BK * 8); const uint unroll_count = UNROLL_K / BK; // Detect a fast path where all loads are entirely in bounds and no clamping is required if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 && -#if QUANT_K == 1 +#if !defined(MULMAT_QUANT) && QUANT_K == 1 (stride_a % 8) == 0 && #endif (stride_b % 8) == 0) { // Hint to the compiler that values are aligned (want 16B alignment) start_k &= ~(START_ALIGN_K-1); stride_b &= ~7; -#if QUANT_K == 1 +#if !defined(MULMAT_QUANT) && QUANT_K == 1 stride_a &= ~7; #endif @@ -551,10 +640,10 @@ void main() { [[dont_unroll]] for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { - if ((block_k % QUANT_K) == 0) { + if ((block_k % qk) == 0) { store_scales(tid); } - if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) { + if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) { fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false); } @@ -588,17 +677,17 @@ void main() { coopMatPerElementNV(mat_d, mat_d, perElemOpD, ir, ic); return; } - if (enable_smaller_matrices && ic * BN + BNover2 >= _ne1) { + if (ic * BN + BNover2 >= _ne1) { coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> sum; sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(0.0); [[dont_unroll]] for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { - if ((block_k % QUANT_K) == 0) { + if ((block_k % qk) == 0) { store_scales(tid); } - if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) { + if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) { fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false); } @@ -639,10 +728,10 @@ void main() { [[dont_unroll]] for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { - if ((block_k % QUANT_K) == 0) { + if ((block_k % qk) == 0) { store_scales(tid); } - if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) { + if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) { fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl index 7d852dced8ab..588fb73546a3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl @@ -20,651 +20,704 @@ void store_a(uint m, uint k_pair, FLOAT_TYPEV2 value) { void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uint idx_m, const uint block, const uint end_k) { #if defined(DATA_A_F32) || defined(DATA_A_F16) #if LOAD_VEC_A == 8 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]); - store_a(col, k_pair, aa[0].xy); - store_a(col, k_pair + 1, aa[0].zw); - store_a(col, k_pair + 2, aa[1].xy); - store_a(col, k_pair + 3, aa[1].zw); - return; - } + if (ALIGNED != 0) { + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]); + store_a(col, k_pair, aa[0].xy); + store_a(col, k_pair + 1, aa[0].zw); + store_a(col, k_pair + 2, aa[1].xy); + store_a(col, k_pair + 3, aa[1].zw); + return; + } #elif LOAD_VEC_A == 4 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]); - store_a(col, k_pair, aa.xy); - store_a(col, k_pair + 1, aa.zw); - return; - } + if (ALIGNED != 0) { + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]); + store_a(col, k_pair, aa.xy); + store_a(col, k_pair + 1, aa.zw); + return; + } #endif - const uint idx = pos_a + col * p.stride_a + row * 2; - if (idx_m < p.M && block + row * 2 + 1 < end_k) { - store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], - data_a_scalar[idx + 1])); - } else if (idx_m < p.M && block + row * 2 < end_k) { - store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], 0.0f)); - } else { - store_a(col, row, FLOAT_TYPEV2(0.0f)); - } + const uint idx = pos_a + col * p.stride_a + row * 2; + if (idx_m < p.M && block + row * 2 + 1 < end_k) { + store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], + data_a_scalar[idx + 1])); + } else if (idx_m < p.M && block + row * 2 < end_k) { + store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], 0.0f)); + } else { + store_a(col, row, FLOAT_TYPEV2(0.0f)); + } #elif defined(DATA_A_BF16) #if LOAD_VEC_A == 4 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx])); - store_a(col, k_pair, aa.xy); - store_a(col, k_pair + 1, aa.zw); - return; - } + if (ALIGNED != 0) { + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx])); + store_a(col, k_pair, aa.xy); + store_a(col, k_pair + 1, aa.zw); + return; + } #endif - const uint idx = pos_a + col * p.stride_a + row * 2; - if (idx_m < p.M && block + row * 2 + 1 < end_k) { - store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), - TO_FLOAT_TYPE(data_a_scalar[idx + 1]))); - } else if (idx_m < p.M && block + row * 2 < end_k) { - store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f)); - } else { - store_a(col, row, FLOAT_TYPEV2(0.0f)); - } -#elif defined(DATA_A_Q4_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const float d = float(data_a_packed16[ib].d); - const uint vui = uint(data_a_packed16[ib].qs[2*iqs]) | (uint(data_a_packed16[ib].qs[2*iqs + 1]) << 16); - const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d; - const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d; - - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); -#elif defined(DATA_A_Q4_1) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const vec2 dm = vec2(data_a_packed32[ib].dm); - const uint vui = data_a_packed32[ib].qs[iqs]; - const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y; - const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y; - - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); -#elif defined(DATA_A_Q5_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 8; - const uint iqs = idx & 0x07; - - const float d = float(data_a_packed16[ib].d); - const uint uint_qh = uint(data_a_packed16[ib].qh[1]) << 16 | uint(data_a_packed16[ib].qh[0]); - const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10); - const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10); - - const uint vui = uint(data_a_packed16[ib].qs[iqs]); - const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d; - store_a(col, row, FLOAT_TYPEV2(v.xz)); - store_a(col, row + 8, FLOAT_TYPEV2(v.yw)); -#elif defined(DATA_A_Q5_1) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const vec2 dm = vec2(data_a_packed32[ib].dm); - const uint uint_qh = data_a_packed32[ib].qh; - const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10); - const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10); - const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10); - const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10); - - const uint vui = data_a_packed32[ib].qs[iqs]; - const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y; - const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y; - - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, FLOAT_TYPEV2(v0.xz)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v1.xz)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v0.yw)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.yw)); -#elif defined(DATA_A_Q8_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 8; - const uint iqs = idx & 0x07; - - const float d = float(data_a_packed16[ib].d); - const i8vec2 v0 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs])).xy; // vec4 used due to #12147 - const i8vec2 v1 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs + 1])).xy; - const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d; - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); -#elif defined(DATA_A_Q1_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 16; - const uint iqs = idx & 0xfu; - - const float d = float(data_a[ib].d); - const uint bits = uint(data_a[ib].qs[iqs]); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d)); - store_a(col, k_pair + 1, FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d)); - store_a(col, k_pair + 2, FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d)); - store_a(col, k_pair + 3, FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d)); -#elif defined(DATA_A_Q2_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 16; - const uint iqs = idx & 0xfu; - - const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib].d); - const uint bits = uint(data_a[ib].qs[iqs]); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f))); - store_a(col, k_pair + 1, d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f))); -#elif defined(DATA_A_Q2_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 - - const uint qsi = (iqs / 64) * 16 + (iqs % 16); // 0..15 - const uint scalesi = iqs / 8; // 0..15 - const uint qsshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 - - const vec4 qs = vec4(unpack8((data_a_packed32[ib].qs[qsi / 2] >> qsshift) & 0x03030303)); - const uint scales = data_a[ib].scales[scalesi]; - const vec2 dm = vec2(data_a[ib].dm); - - const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); -#elif defined(DATA_A_TQ2_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = (idx % 128) * 2; // elem 0,2,4..254 - - const uint qsi = (iqs / 128) * 32 + (iqs % 32); // byte pair start - const uint shift = 2 * ((iqs % 128) / 32); // 0,2,4,6 - - const uvec2 qs = uvec2(data_a[ib].qs[qsi], data_a[ib].qs[qsi + 1]); - const float d = float(data_a[ib].d); - - const vec2 v = d * (vec2((qs >> shift) & 3) - 1.0); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); -#elif defined(DATA_A_Q3_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = idx % 128; // 0..127 - - const uint n = iqs / 64; // 0,1 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..62 - const uint hmi = (iqs % 16) * 2; // 0,2,4..30 - const uint j = (iqs % 64) / 4; // 0..3 - const uint is = iqs / 8; // 0..15 - const uint halfsplit = ((iqs % 64) / 16); // 0,1,2,3 - const uint qsshift = halfsplit * 2; // 0,2,4,6 - - const int8_t us = int8_t(((data_a[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF) - | (((data_a[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4)); - const float dl = float(data_a[ib].d) * float(us - 32); - - const vec2 qs = vec2(unpack8((uint(data_a_packed16[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy); - const vec2 hm = vec2(unpack8(((uint(data_a_packed16[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy); - - store_a(col, row * LOAD_VEC_A / 2, FLOAT_TYPEV2(dl * (qs.x - hm.x), - dl * (qs.y - hm.y))); -#elif defined(DATA_A_Q4_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 - - const uint n = iqs / 32; // 0,1,2,3 - const uint b = (iqs % 32) / 16; // 0,1 - const uint is = 2 * n + b; // 0..7 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 - - const vec2 loadd = vec2(data_a[ib].dm); - - const uvec3 scales = uvec3(data_a_packed32[ib].scales[0], - data_a_packed32[ib].scales[1], - data_a_packed32[ib].scales[2]); - const uint scalesoffs = (is & 3) * 8; + const uint idx = pos_a + col * p.stride_a + row * 2; + if (idx_m < p.M && block + row * 2 + 1 < end_k) { + store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), + TO_FLOAT_TYPE(data_a_scalar[idx + 1]))); + } else if (idx_m < p.M && block + row * 2 < end_k) { + store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f)); + } else { + store_a(col, row, FLOAT_TYPEV2(0.0f)); + } +#elif defined(DATA_A_IQ1_S) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; - const uint scidx0 = (is < 4) ? 0 : 2; - const uint scidxshift0 = scalesoffs; - const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - const uint mbidx0 = (is < 4) ? 1 : 2; - const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; - const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - - const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); - const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); - - const float d = loadd.x * sc; - const float m = -loadd.y * mbyte; + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 32; - const vec4 q = vec4(unpack8((data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F)); + const float d = float(data_a[ib].d); + const uint qh = data_a[ib].qh[ib32]; + const uint qs = data_a[ib].qs[ib8]; + const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1); + const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA; + const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]); - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); - store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); -#elif defined(DATA_A_Q5_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + [[unroll]] for (int k = 0; k < 4; ++k) { + store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), + dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 + } +#elif defined(DATA_A_IQ1_M) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib8 = idx % 32; + const uint ib16 = ib8 / 2; + + const uint16_t[4] scales = data_a[ib].scales; + const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12; + const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x); + const uint sc = scales[ib8 / 8]; + const uint qs = data_a[ib].qs[ib8]; + const uint qh = data_a[ib].qh[ib16] >> (4 * (ib8 & 1)); + const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1); + const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA; + const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]); + + [[unroll]] for (int k = 0; k < 4; ++k) { + store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), + dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - const uint n = iqs / 32; // 0,1,2,3 - const uint b = (iqs % 32) / 16; // 0,1 - const uint is = 2 * n + b; // 0..7 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 - const uint qhi = (iqs % 16) * 2; // 0,2,4..30 + } +#elif defined(DATA_A_IQ2_XXS) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 4; + + const float d = float(data_a[ib].d); + const uint qs = data_a[ib].qs[8 * ib32 + ib8]; + const uint signs = pack32(u8vec4( + data_a[ib].qs[8*ib32 + 4], + data_a[ib].qs[8*ib32 + 5], + data_a[ib].qs[8*ib32 + 6], + data_a[ib].qs[8*ib32 + 7] + )); + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); + const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); + const uint sign = sign7 | (bitCount(sign7) << 7); + const uvec2 grid = iq2xxs_grid[qs]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); - const vec2 loadd = vec2(data_a[ib].dm); +#elif defined(DATA_A_IQ2_XS) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; - const uvec3 scales = uvec3(data_a_packed32[ib].scales[0], - data_a_packed32[ib].scales[1], - data_a_packed32[ib].scales[2]); - const uint scalesoffs = (is & 3) * 8; + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 4; - const uint scidx0 = (is < 4) ? 0 : 2; - const uint scidxshift0 = scalesoffs; - const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - const uint mbidx0 = (is < 4) ? 1 : 2; - const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; - const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + const float d = float(data_a[ib].d); + const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); + const uint qs = data_a[ib].qs[4 * ib32 + ib8]; + const uint sign7 = qs >> 9; + const uint sign = sign7 | (bitCount(sign7) << 7); + const uvec2 grid = iq2xs_grid[qs & 511]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); - const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); - const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); - - const float d = loadd.x * sc; - const float m = -loadd.y * mbyte; - - const uint qs = (data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F; - const uint qh = ((data_a_packed32[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4; - const vec4 q = vec4(unpack8(qs | qh)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); - store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); -#elif defined(DATA_A_Q6_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = idx % 128; // 0..127 - - const uint n = iqs / 64; // 0,1 - const uint b = ((iqs % 64) / 32) * 4; // 0,4 - const uint is_b = (iqs % 16) / 8; // 0,1 - const uint qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 - const uint is = 8 * n + qhshift + is_b; // 0..15 - const uint qsi = n * 32 + (iqs % 32); // 0..63 - const uint qhi = n * 16 + (iqs % 16); // 0..31 - - const float dscale = float(data_a[ib].d) * float(data_a[ib].scales[is]); + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); - const uint ql = (uint(data_a_packed16[ib].ql[qsi]) >> b) & 0x0F0F; - const uint qh = (uint(data_a_packed16[ib].qh[qhi]) >> qhshift) & 0x0303; - const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale; + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); - store_a(col, row * LOAD_VEC_A / 2, FLOAT_TYPEV2(q.x, q.y)); -#elif defined(DATA_A_IQ1_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 32; - - const float d = float(data_a[ib].d); - const uint qh = data_a[ib].qh[ib32]; - const uint qs = data_a[ib].qs[ib8]; - const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1); - const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA; - const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]); - - const uint k_pair = row * LOAD_VEC_A / 2; - [[unroll]] for (int k = 0; k < 4; ++k) { - store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), - dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - } -#elif defined(DATA_A_IQ1_M) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib8 = idx % 32; - const uint ib16 = ib8 / 2; - - const uint16_t[4] scales = data_a[ib].scales; - const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12; - const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x); - const uint sc = scales[ib8 / 8]; - const uint qs = data_a[ib].qs[ib8]; - const uint qh = data_a[ib].qh[ib16] >> (4 * (ib8 & 1)); - const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1); - const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA; - const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]); - - const uint k_pair = row * LOAD_VEC_A / 2; - [[unroll]] for (int k = 0; k < 4; ++k) { - store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), - dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - } -#elif defined(DATA_A_IQ2_XXS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 4; - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[8 * ib32 + ib8]; - const uint signs = pack32(u8vec4( - data_a[ib].qs[8*ib32 + 4], - data_a[ib].qs[8*ib32 + 5], - data_a[ib].qs[8*ib32 + 6], - data_a[ib].qs[8*ib32 + 7] - )); - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); - const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); - const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2xxs_grid[qs]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); -#elif defined(DATA_A_IQ2_XS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 4; // 0..3 - - const float d = float(data_a[ib].d); - const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uint qs = data_a[ib].qs[4 * ib32 + ib8]; - const uint sign7 = qs >> 9; - const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2xs_grid[qs & 511]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); #elif defined(DATA_A_IQ2_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib8 = idx % 32; // 0..31 - const uint ib32 = ib8 / 4; // 0..7 - - const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; - const uint qs = data_a[ib].qs[ib8]; - const uint qh = data_a[ib].qh[ib32]; - const uint qhshift = 2 * (ib8 % 4); - const uint sign = data_a[ib].qs[QUANT_K / 8 + ib8]; - - const float d = float(data_a[ib].d); - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib8 = idx % 32; + const uint ib32 = ib8 / 4; + + const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; + const uint qs = data_a[ib].qs[ib8]; + const uint qh = data_a[ib].qh[ib32]; + const uint qhshift = 2 * (ib8 % 4); + const uint sign = data_a[ib].qs[QUANT_K_IQ2_S / 8 + ib8]; + + const float d = float(data_a[ib].d); + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); + const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); + #elif defined(DATA_A_IQ3_XXS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = idx % 64; // 0..63 - const uint is = QUANT_K / 4 + 4 * (iqs / 8); // 8 values - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[iqs]; - const uint signs = pack32(u16vec2( - data_a_packed16[ib].qs[is/2], - data_a_packed16[ib].qs[is/2+1] - )); - const float db = d * 0.5 * (0.5 + (signs >> 28)); - const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); - const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); - const uint grid = iq3xxs_grid[qs]; - const vec4 v = db * vec4(unpack8(grid)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, - (sign & 2) != 0 ? -v.y : v.y)); - store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, - (sign & 8) != 0 ? -v.w : v.w)); + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 64; + const uint iqs = idx % 64; + const uint is = QUANT_K_IQ3_XXS / 4 + 4 * (iqs / 8); + + const float d = float(data_a[ib].d); + const uint qs = data_a[ib].qs[iqs]; + const uint signs = pack32(u16vec2( + data_a_packed16[ib].qs[is/2], + data_a_packed16[ib].qs[is/2+1] + )); + const float db = d * 0.5 * (0.5 + (signs >> 28)); + const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); + const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); + const uint grid = iq3xxs_grid[qs]; + const vec4 v = db * vec4(unpack8(grid)); + + store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, + (sign & 2) != 0 ? -v.y : v.y)); + + store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, + (sign & 8) != 0 ? -v.w : v.w)); + #elif defined(DATA_A_IQ3_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = idx % 64; // 0..63 - const uint iqh = iqs / 8; - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[iqs]; - const uint qh = data_a[ib].qh[iqh]; - const int8_t sign = int8_t(data_a[ib].signs[iqs / 2] >> (4 * (idx % 2))); - const uint scale = data_a[ib].scales[iqs / 16]; - const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign))); - const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; - const vec4 v = db * vec4(unpack8(grid)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, - (sign & 2) != 0 ? -v.y : v.y)); - store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, - (sign & 8) != 0 ? -v.w : v.w)); + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 64; + const uint iqs = idx % 64; + const uint iqh = iqs / 8; + + const float d = float(data_a[ib].d); + const uint qs = data_a[ib].qs[iqs]; + const uint qh = data_a[ib].qh[iqh]; + const int8_t sign = int8_t(data_a[ib].signs[iqs / 2] >> (4 * (idx % 2))); + const uint scale = data_a[ib].scales[iqs / 16]; + const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign))); + const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); + const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; + const vec4 v = db * vec4(unpack8(grid)); + + store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, + (sign & 2) != 0 ? -v.y : v.y)); + + store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, + (sign & 8) != 0 ? -v.w : v.w)); + #elif defined(DATA_A_IQ4_XS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 4; + + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint iq = 4 * ib32 + (idx % 4); - const uint ib = idx / 64; // 4 values per idx - const uint ib32 = (idx % 64) / 8; // 0..7 - const uint iq = 4 * ib32 + (idx % 4); + const uint sl = (data_a[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; + const uint sh = ((data_a[ib].scales_h) >> (2 * ib32)) & 3; + const float d = float(data_a[ib].d); + const float dl = d * float(int(sl | (sh << 4)) - 32); + const uint vui = uint(data_a_packed32[ib].qs[iq]); - const uint sl = (data_a[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; - const uint sh = ((data_a[ib].scales_h) >> (2 * ib32)) & 3; - const uint qshift = idx & 4; - u8vec4 qs = unpack8((uint(data_a_packed32[ib].qs[iq]) >> qshift) & 0x0F0F0F0F); + const u8vec4 qs0 = unpack8( vui & 0x0F0F0F0F); + const u8vec4 qs1 = unpack8((vui >> 4) & 0x0F0F0F0F); + const vec4 v0 = dl * vec4(kvalues_iq4nl[qs0.x], kvalues_iq4nl[qs0.y], kvalues_iq4nl[qs0.z], kvalues_iq4nl[qs0.w]); + const vec4 v1 = dl * vec4(kvalues_iq4nl[qs1.x], kvalues_iq4nl[qs1.y], kvalues_iq4nl[qs1.z], kvalues_iq4nl[qs1.w]); - const float d = float(data_a[ib].d); - const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]); + store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); + store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); #elif defined(DATA_A_IQ4_NL) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 4; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; - const uint ib = idx / 8; - const uint iqs = idx & 0x07; + const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d); + const uint vui = uint(data_a_packed16[ib].qs[iqs]); - const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d); - const uint vui = uint(data_a_packed16[ib].qs[iqs]); + store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF], + kvalues_iq4nl[bitfieldExtract(vui, 8, 4)])); + + store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)], + kvalues_iq4nl[vui >> 12])); - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF], - kvalues_iq4nl[bitfieldExtract(vui, 8, 4)])); - store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)], - kvalues_iq4nl[vui >> 12])); #elif defined(DATA_A_MXFP4) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 4; - const uint ib = idx / 8; - const uint iqs = (idx & 0x07) * 2; + const uint ib = idx / 8; + const uint iqs = (idx & 0x07) * 2; - const uint vui = uint(data_a[ib].qs[iqs]); - const uint vui2 = uint(data_a[ib].qs[iqs+1]); + const uint vui = uint(data_a[ib].qs[iqs]); + const uint vui2 = uint(data_a[ib].qs[iqs+1]); #ifdef USE_OCP_FP4 - const float d = e8m0_to_fp32(data_a[ib].e); - const u8vec2 packed = u8vec2(vui, vui2); - store_a(col, row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d)); - store_a(col, row + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d)); + const float d = e8m0_to_fp32(data_a[ib].e); + const u8vec2 packed = u8vec2(vui, vui2); + store_a(col, k_pair, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d)); #else - const float d = e8m0_to_fp32(data_a[ib].e) * 0.5; - store_a(col, row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, - kvalues_mxfp4[vui2 & 0xF] * d)); - store_a(col, row + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, - kvalues_mxfp4[vui2 >> 4] * d)); + const float d = e8m0_to_fp32(data_a[ib].e) * 0.5; + store_a(col, k_pair, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, + kvalues_mxfp4[vui2 & 0xF] * d)); + + store_a(col, k_pair + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, + kvalues_mxfp4[vui2 >> 4] * d)); + #endif #elif defined(DATA_A_NVFP4) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint ib = idx / 16u; - const uint sub = (idx & 0xC) >> 2; - const uint iqs = (idx & 0xF) * 2; - const uint vui = uint(data_a[ib].qs[iqs]); - const uint vui2 = uint(data_a[ib].qs[iqs+1]); - - // lo and hi nibbles are 8 elements apart, which doesn't quite line up with - // how the thread mapping and buf_idx calculation works for other types. - const uint eff_row = (row & 3) + (row & ~3) * 2; + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint eff_row = (row & 3) + (row & ~3) * 2; + + const uint ib = idx / 16u; + const uint sub = (idx & 0xC) >> 2; + const uint iqs = (idx & 0xF) * 2; + const uint vui = uint(data_a[ib].qs[iqs]); + const uint vui2 = uint(data_a[ib].qs[iqs+1]); + #ifdef USE_OCP_FP4 - const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(data_a[ib].d[sub])); - const u8vec2 packed = u8vec2(vui, vui2); - store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d); - store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d); + const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(data_a[ib].d[sub])); + const u8vec2 packed = u8vec2(vui, vui2); + store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d); + store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d); #else - const float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5; - store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, - kvalues_mxfp4[vui2 & 0xF] * d)); - store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, - kvalues_mxfp4[vui2 >> 4] * d)); + const float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5; + store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, + kvalues_mxfp4[vui2 & 0xF] * d)); + store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, + kvalues_mxfp4[vui2 >> 4] * d)); #endif +#else + if (MmTypeA == GGML_TYPE_Q4_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const float d = float(a_q4_0_p16.data[ib].d); + const uint vui = uint(a_q4_0_p16.data[ib].qs[2*iqs]) | (uint(a_q4_0_p16.data[ib].qs[2*iqs + 1]) << 16); + const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d; + const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d; + + store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); + store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); + } else if (MmTypeA == GGML_TYPE_Q4_1) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const vec2 dm = vec2(a_q4_1_p32.data[ib].dm); + const uint vui = a_q4_1_p32.data[ib].qs[iqs]; + const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y; + const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y; + + store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); + store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); + } else if (MmTypeA == GGML_TYPE_Q5_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; + + const float d = float(a_q5_0_p16.data[ib].d); + const uint uint_qh = uint(a_q5_0_p16.data[ib].qh[1]) << 16 | uint(a_q5_0_p16.data[ib].qh[0]); + const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10); + const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10); + + const uint vui = uint(a_q5_0_p16.data[ib].qs[iqs]); + const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d; + + store_a(col, k_pair, FLOAT_TYPEV2(v.xz)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v.yw)); + } else if (MmTypeA == GGML_TYPE_Q5_1) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const vec2 dm = vec2(a_q5_1_p32.data[ib].dm); + const uint uint_qh = a_q5_1_p32.data[ib].qh; + const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10); + const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10); + const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10); + const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10); + + const uint vui = a_q5_1_p32.data[ib].qs[iqs]; + const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y; + const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y; + + store_a(col, k_pair, FLOAT_TYPEV2(v0.xz)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v1.xz)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v0.yw)); + store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.yw)); + } else if (MmTypeA == GGML_TYPE_Q8_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; + + const float d = float(a_q8_0_p16.data[ib].d); + const i8vec2 v0 = unpack8(int32_t(a_q8_0_p16.data[ib].qs[2*iqs])).xy; // vec4 used due to #12147 + const i8vec2 v1 = unpack8(int32_t(a_q8_0_p16.data[ib].qs[2*iqs + 1])).xy; + const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d; + + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); + } else if (MmTypeA == GGML_TYPE_Q1_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 16; + const uint iqs = idx & 0xfu; + + const float d = float(a_q1_0.data[ib].d); + const uint bits = uint(a_q1_0.data[ib].qs[iqs]); + + store_a(col, k_pair, FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d)); + store_a(col, k_pair + 1, FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d)); + store_a(col, k_pair + 2, FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d)); + store_a(col, k_pair + 3, FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d)); + } else if (MmTypeA == GGML_TYPE_Q2_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 16; + const uint iqs = idx & 0xfu; + + const FLOAT_TYPE d = FLOAT_TYPE(a_q2_0.data[ib].d); + const uint bits = uint(a_q2_0.data[ib].qs[iqs]); + + store_a(col, k_pair, d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f))); + store_a(col, k_pair + 1, d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f))); + } else if (MmTypeA == GGML_TYPE_Q2_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint iqs = (idx % 64) * 2; // 0,2,4..126 + + const uint qsi = (iqs / 64) * 16 + (iqs % 16); // 0..15 + const uint scalesi = iqs / 8; // 0..15 + const uint qsshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 + + const vec4 qs = vec4(unpack8((a_q2_k_p32.data[ib].qs[qsi / 2] >> qsshift) & 0x03030303)); + const uint scales = a_q2_k.data[ib].scales[scalesi]; + const vec2 dm = vec2(a_q2_k.data[ib].dm); + + const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4); + + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); + } else if (MmTypeA == GGML_TYPE_Q3_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 128; // 2 values per idx + const uint iqs = idx % 128; // 0..127 + + const uint n = iqs / 64; // 0,1 + const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..62 + const uint hmi = (iqs % 16) * 2; // 0,2,4..30 + const uint j = (iqs % 64) / 4; // 0..3 + const uint is = iqs / 8; // 0..15 + const uint halfsplit = ((iqs % 64) / 16); // 0,1,2,3 + const uint qsshift = halfsplit * 2; // 0,2,4,6 + + const int8_t us = int8_t(((a_q3_k.data[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF) + | (((a_q3_k.data[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4)); + const float dl = float(a_q3_k.data[ib].d) * float(us - 32); + + const vec2 qs = vec2(unpack8((uint(a_q3_k_p16.data[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy); + const vec2 hm = vec2(unpack8(((uint(a_q3_k_p16.data[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy); + + store_a(col, k_pair, FLOAT_TYPEV2(dl * (qs.x - hm.x), + dl * (qs.y - hm.y))); + + } else if (MmTypeA == GGML_TYPE_Q4_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint iqs = (idx % 64) * 2; // 0,2,4..126 + + const uint n = iqs / 32; // 0,1,2,3 + const uint b = (iqs % 32) / 16; // 0,1 + const uint is = 2 * n + b; // 0..7 + const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 + + const vec2 loadd = vec2(a_q4_k.data[ib].dm); + + const uvec3 scales = uvec3(a_q4_k_p32.data[ib].scales[0], + a_q4_k_p32.data[ib].scales[1], + a_q4_k_p32.data[ib].scales[2]); + const uint scalesoffs = (is & 3) * 8; + + const uint scidx0 = (is < 4) ? 0 : 2; + const uint scidxshift0 = scalesoffs; + const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + const uint mbidx0 = (is < 4) ? 1 : 2; + const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; + const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + + const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); + const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); + + const float d = loadd.x * sc; + const float m = -loadd.y * mbyte; + + const vec4 q = vec4(unpack8((a_q4_k_p32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F)); + + store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); + store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); + } else if (MmTypeA == GGML_TYPE_Q5_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint iqs = (idx % 64) * 2; // 0,2,4..126 + + const uint n = iqs / 32; // 0,1,2,3 + const uint b = (iqs % 32) / 16; // 0,1 + const uint is = 2 * n + b; // 0..7 + const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 + const uint qhi = (iqs % 16) * 2; // 0,2,4..30 + + const vec2 loadd = vec2(a_q5_k.data[ib].dm); + + const uvec3 scales = uvec3(a_q5_k_p32.data[ib].scales[0], + a_q5_k_p32.data[ib].scales[1], + a_q5_k_p32.data[ib].scales[2]); + const uint scalesoffs = (is & 3) * 8; + + const uint scidx0 = (is < 4) ? 0 : 2; + const uint scidxshift0 = scalesoffs; + const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + const uint mbidx0 = (is < 4) ? 1 : 2; + const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; + const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + + const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); + const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); + + const float d = loadd.x * sc; + const float m = -loadd.y * mbyte; + + const uint qs = (a_q5_k_p32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F; + const uint qh = ((a_q5_k_p32.data[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4; + const vec4 q = vec4(unpack8(qs | qh)); + + store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); + store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); + } else if (MmTypeA == GGML_TYPE_Q6_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 128; // 2 values per idx + const uint iqs = idx % 128; // 0..127 + + const uint n = iqs / 64; // 0,1 + const uint b = ((iqs % 64) / 32) * 4; // 0,4 + const uint is_b = (iqs % 16) / 8; // 0,1 + const uint qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 + const uint is = 8 * n + qhshift + is_b; // 0..15 + const uint qsi = n * 32 + (iqs % 32); // 0..63 + const uint qhi = n * 16 + (iqs % 16); // 0..31 + + const float dscale = float(a_q6_k.data[ib].d) * float(a_q6_k.data[ib].scales[is]); + + const uint ql = (uint(a_q6_k_p16.data[ib].ql[qsi]) >> b) & 0x0F0F; + const uint qh = (uint(a_q6_k_p16.data[ib].qh[qhi]) >> qhshift) & 0x0303; + const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale; + + store_a(col, k_pair, FLOAT_TYPEV2(q.x, q.y)); + } else if (MmTypeA == GGML_TYPE_TQ1_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + + const uint ib = idx / 128; // 2 values per idx + const uint iqs = (idx % 128) * 2; // elem 0,2,4..254 + + const float d = float(a_tq1_0.data[ib].d); + vec2 v; + for (uint kk = 0u; kk < 2u; ++kk) { + const uint e = iqs + kk; + const uint bidx = tq1_0_byte_of(e); + const uint qbyte = uint(bidx < 48u ? a_tq1_0.data[ib].qs[bidx] + : a_tq1_0.data[ib].qh[bidx - 48u]); + v[kk] = d * (float(tq1_0_trit(qbyte, tq1_0_digit_of(e))) - 1.0); + } + + const uint k_pair = row * mm_load_vec_a() / 2; + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + } else if (MmTypeA == GGML_TYPE_TQ2_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + + const uint ib = idx / 128; // 2 values per idx + const uint iqs = (idx % 128) * 2; // elem 0,2,4..254 + + const uint qsi = (iqs / 128) * 32 + (iqs % 32); // byte pair start + const uint shift = 2 * ((iqs % 128) / 32); // 0,2,4,6 + + const uvec2 qs = uvec2(a_tq2_0.data[ib].qs[qsi], a_tq2_0.data[ib].qs[qsi + 1]); + const float d = float(a_tq2_0.data[ib].d); + + const vec2 v = d * (vec2((qs >> shift) & 3) - 1.0); + + const uint k_pair = row * mm_load_vec_a() / 2; + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + } #endif } #if !defined(MUL_MAT_ID) void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint idx_n, const uint block, const uint end_k) { #if LOAD_VEC_B == 8 - if (ALIGNED != 0) { - // Not supported for b_type bf16 because bf16mat2x4 does not exist - const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; - FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); - buf_b[buf_idx + 0] = bb[0].xy; - buf_b[buf_idx + 1] = bb[0].zw; - buf_b[buf_idx + 2] = bb[1].xy; - buf_b[buf_idx + 3] = bb[1].zw; - return; - } + if (ALIGNED != 0) { + // Not supported for b_type bf16 because bf16mat2x4 does not exist + const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); + buf_b[buf_idx + 0] = bb[0].xy; + buf_b[buf_idx + 1] = bb[0].zw; + buf_b[buf_idx + 2] = bb[1].xy; + buf_b[buf_idx + 3] = bb[1].zw; + return; + } #elif LOAD_VEC_B == 4 - if (ALIGNED != 0) { - const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + if (ALIGNED != 0) { + const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; #if defined(DATA_B_BF16) - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); #else - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); #endif - buf_b[buf_idx + 0] = bb.xy; - buf_b[buf_idx + 1] = bb.zw; - return; - } + buf_b[buf_idx + 0] = bb.xy; + buf_b[buf_idx + 1] = bb.zw; + return; + } #endif - const uint idx = pos_b + col * p.stride_b + row * 2; - const uint buf_idx = col * SHMEM_STRIDE + row; - if (idx_n < p.N && block + row * 2 + 1 < end_k) { - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), - TO_FLOAT_TYPE(data_b_scalar[idx + 1])); - } else if (idx_n < p.N && block + row * 2 < end_k) { - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); - } else { - buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); - } + const uint idx = pos_b + col * p.stride_b + row * 2; + const uint buf_idx = col * SHMEM_STRIDE + row; + if (idx_n < p.N && block + row * 2 + 1 < end_k) { + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), + TO_FLOAT_TYPE(data_b_scalar[idx + 1])); + } else if (idx_n < p.N && block + row * 2 < end_k) { + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); + } else { + buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); + } } #else void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint ic, const uint _ne1, const uint block, const uint end_k) { #if LOAD_VEC_B == 8 - if (ALIGNED != 0) { - // Not supported for b_type bf16 because bf16mat2x4 does not exist - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; - FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); - buf_b[buf_idx + 0] = bb[0].xy; - buf_b[buf_idx + 1] = bb[0].zw; - buf_b[buf_idx + 2] = bb[1].xy; - buf_b[buf_idx + 3] = bb[1].zw; - return; - } + if (ALIGNED != 0) { + // Not supported for b_type bf16 because bf16mat2x4 does not exist + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); + buf_b[buf_idx + 0] = bb[0].xy; + buf_b[buf_idx + 1] = bb[0].zw; + buf_b[buf_idx + 2] = bb[1].xy; + buf_b[buf_idx + 3] = bb[1].zw; + return; + } #elif LOAD_VEC_B == 4 - if (ALIGNED != 0) { - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + if (ALIGNED != 0) { + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; #if defined(DATA_B_BF16) - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); #else - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); #endif - buf_b[buf_idx + 0] = bb.xy; - buf_b[buf_idx + 1] = bb.zw; - return; - } + buf_b[buf_idx + 0] = bb.xy; + buf_b[buf_idx + 1] = bb.zw; + return; + } #endif - const uint row_i = ic * BN + col; - const uint buf_idx = col * SHMEM_STRIDE + row; - if (row_i < _ne1 && block + row * 2 + 1 < end_k) { - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), - TO_FLOAT_TYPE(data_b_scalar[idx + 1])); - } else if (row_i < _ne1 && block + row * 2 < end_k) { - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); - } else { - buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); - } + const uint row_i = ic * BN + col; + const uint buf_idx = col * SHMEM_STRIDE + row; + if (row_i < _ne1 && block + row * 2 + 1 < end_k) { + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), + TO_FLOAT_TYPE(data_b_scalar[idx + 1])); + } else if (row_i < _ne1 && block + row * 2 < end_k) { + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); + } else { + buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); + } } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp index 1fbcbf6c9332..67908e704879 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp @@ -6,6 +6,8 @@ #extension GL_EXT_integer_dot_product : require +#define KVALUES_IQ4NL_I8 + #ifdef FLOAT16 #extension GL_EXT_shader_explicit_arithmetic_types_float16 : require #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl index 24da4f715f83..136d5b74532e 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl @@ -217,6 +217,41 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { } #endif +#if defined(DATA_A_IQ4_XS) +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { + const uint ib_k = ib / 8; + const uint ib32 = ib % 8; + const uint32_t vui = data_a_packed32[ib_k].qs[4 * ib32 + iqs]; + const i32vec2 qs = iq4nl_to_i8x8(vui); + + buf_a[buf_ib].qs[iqs ] = qs.x; + buf_a[buf_ib].qs[iqs + 4] = qs.y; + + if (iqs == 0) { + const uint sl = (data_a_packed32[ib_k].scales_l >> (4 * ib32)) & 0xF; + const uint sh = (data_a_packed32[ib_k].scales_h >> (2 * ib32)) & 3; + buf_a[buf_ib].d = FLOAT_TYPE(float(data_a[ib_k].d) * float(int(sl | (sh << 4)) - 32)); + } +} + +void block_a_to_registers(const uint reg_ib, const uint buf_ib) { + cache_a[reg_ib].d = buf_a[buf_ib].d; + + [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) { + cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs]; + } +} + +ACC_TYPE mmq_dot_product(const uint ib_a) { + int32_t q_sum = 0; + [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) { + q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]); + } + + return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum)); +} +#endif + // For k-quants, ib and iqs still assume 32-wide blocks, but k-quants are 256-wide // iqs still refers to a 32-bit integer, meaning 0..7 for 32-wide quants #if defined(DATA_A_Q2_K) @@ -454,6 +489,55 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { } #endif +#if defined(DATA_A_IQ3_S) +// 2-byte loads for IQ3_S blocks (110 bytes) +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { + const uint ib_k = ib / 8; + const uint ib32 = ib % 8; + + // grid indices for qs[2 * iqs] and qs[2 * iqs + 1] + const uint qs = uint(data_a_packed16[ib_k].qs[ib32 * 4 + iqs]); + // their two high index bits + const uint qh = uint(data_a_packed16[ib_k].qh[ib32 / 2]) >> ((ib32 & 1) * 8 + 2 * iqs); + // one sign bit per value, 8 values + const uint signs = uint(data_a_packed16[ib_k].signs[ib32 * 2 + iqs / 2]) >> ((iqs & 1) * 8); + + // grid holds 4 values of 1..15, one per byte + const ivec4 vals0 = ivec4(unpack8(iq3s_grid[( qs & 0xFF) | ((qh & 1) << 8)])); + const ivec4 vals1 = ivec4(unpack8(iq3s_grid[((qs >> 8) & 0xFF) | ((qh & 2) << 7)])); + + // negate with (v ^ -s) - -s to avoid branches + const ivec4 m0 = -(ivec4(signs, signs >> 1, signs >> 2, signs >> 3) & 1); + const ivec4 m1 = -(ivec4(signs >> 4, signs >> 5, signs >> 6, signs >> 7) & 1); + + buf_a[buf_ib].qs[2 * iqs ] = pack32(i8vec4((vals0 ^ m0) - m0)); + buf_a[buf_ib].qs[2 * iqs + 1] = pack32(i8vec4((vals1 ^ m1) - m1)); + + if (iqs == 0) { + const uint scale = (uint(data_a_packed16[ib_k].scales[ib32 / 4]) >> ((ib32 & 3) * 4)) & 0xF; + + buf_a[buf_ib].d = FLOAT_TYPE(float(data_a_packed16[ib_k].d) * float(1 + 2 * scale)); + } +} + +void block_a_to_registers(const uint reg_ib, const uint buf_ib) { + cache_a[reg_ib].d = buf_a[buf_ib].d; + + [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) { + cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs]; + } +} + +ACC_TYPE mmq_dot_product(const uint ib_a) { + int32_t q_sum = 0; + [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) { + q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]); + } + + return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum)); +} +#endif + void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { if (is_in_bounds) { const uint ib_outer = ib / 4; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl index 2b7adcb6c2fc..56784cff38a2 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl @@ -53,12 +53,24 @@ struct block_a_cache { int32_t qs[8]; FLOAT_TYPE dm; }; +#elif defined(DATA_A_IQ4_XS) +#define QUANT_R_MMQ 2 +struct block_a_cache { + int32_t qs[8]; + FLOAT_TYPE d; +}; #elif defined(DATA_A_MXFP4) #define QUANT_R_MMQ 2 struct block_a_cache { int32_t qs[8]; FLOAT_TYPE d; }; +#elif defined(DATA_A_IQ3_S) +#define QUANT_R_MMQ 2 +struct block_a_cache { + int32_t qs[8]; + FLOAT_TYPE d; +}; #elif defined(DATA_A_Q2_K) #define QUANT_R_MMQ 4 struct block_a_cache { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp index 55b89f19a7a8..ee813842c069 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp @@ -27,12 +27,24 @@ layout (binding = 6) readonly buffer R_I {uvec2 rope_data_i[];}; // indices for #define GGML_ROPE_TYPE_MROPE 8 #define GGML_ROPE_TYPE_VISION 24 +#elif RMS_NORM_ADD_FUSION + +layout (binding = 3) readonly buffer C {float data_c[];}; +layout (binding = 4) readonly buffer E {float data_e[];}; + +#elif RMS_NORM_SET_ROWS_FUSION + +layout (binding = 3) readonly buffer I {uvec2 data_i[];}; + #endif #extension GL_EXT_control_flow_attributes : enable #define BLOCK_SIZE 512 layout (constant_id = 1) const bool do_multiply = false; +#if RMS_NORM_ADD_FUSION +layout (constant_id = 2) const bool do_post_multiply = false; +#endif layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; @@ -57,6 +69,8 @@ void rms_norm(uint num_iters) { #if RMS_NORM_ROPE_FUSION // Per-row offset in shared memory uint32_t d_offset = 0; +#elif RMS_NORM_SET_ROWS_FUSION + uint32_t d_offset = data_i[channel].x*p.nb21 + row*ncols + get_doffset(); #else uint32_t d_offset = ((samp*nchannels + channel)*nrows + row)*ncols + get_doffset(); #endif @@ -91,14 +105,28 @@ void rms_norm(uint num_iters) { if (col >= ncols) { continue; } - data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)])); + FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]); +#if RMS_NORM_ADD_FUSION + value += FLOAT_TYPE(data_c[d_offset + col]); + if (do_post_multiply) { + value *= FLOAT_TYPE(data_e[0]); + } +#endif + data_d[d_offset + col] = D_TYPE(value); } } else { [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { if (col >= ncols) { continue; } - data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col])); + FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]); +#if RMS_NORM_ADD_FUSION + value += FLOAT_TYPE(data_c[d_offset + col]); + if (do_post_multiply) { + value *= FLOAT_TYPE(data_e[0]); + } +#endif + data_d[d_offset + col] = D_TYPE(value); } } } else { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp index 4618b2c7e8a1..cf7ab21f261d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp @@ -10,11 +10,19 @@ #define BLOCK_SIZE 128 layout (constant_id = 1) const bool do_multiply = false; +#if RMS_NORM_ADD_FUSION +layout (constant_id = 2) const bool do_post_multiply = false; +#endif layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; layout (binding = 3, std430) readonly buffer PartialsBuf {float partial_sums[];}; +#if RMS_NORM_ADD_FUSION +layout (binding = 4) readonly buffer C {float data_c[];}; +layout (binding = 5) readonly buffer E {float data_e[];}; +#endif + shared FLOAT_TYPE sumsh[BLOCK_SIZE]; void main() { @@ -55,9 +63,23 @@ void main() { if (do_multiply) { if (ncols > p.ne10) { - data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)])); + FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]); +#if RMS_NORM_ADD_FUSION + value += FLOAT_TYPE(data_c[d_offset + col]); + if (do_post_multiply) { + value *= FLOAT_TYPE(data_e[0]); + } +#endif + data_d[d_offset + col] = D_TYPE(value); } else { - data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col])); + FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]); +#if RMS_NORM_ADD_FUSION + value += FLOAT_TYPE(data_c[d_offset + col]); + if (do_post_multiply) { + value *= FLOAT_TYPE(data_e[0]); + } +#endif + data_d[d_offset + col] = D_TYPE(value); } } else { data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col])); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index 24c6ba1a608a..5be844e8e1d4 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -332,6 +332,41 @@ struct block_q2_K_packed32 #define DATA_A_QUANT_K #endif +#define QUANT_K_TQ1_0 256 + +// TQ1_0: base-3 packed trits, 5 per byte in `qs` (48B) and 4 in `qh` (4B). +struct block_tq1_0 +{ + uint8_t qs[(QUANT_K_TQ1_0 - 4 * QUANT_K_TQ1_0 / 64) / 5]; + uint8_t qh[QUANT_K_TQ1_0 / 64]; + float16_t d; +}; + +// Element e in [0,255] -> its packed byte (0..47 qs, 48..51 qh) and digit. +uint tq1_0_byte_of(uint e) { + return e < 160u ? (e % 32u) + : e < 240u ? 32u + ((e - 160u) % 16u) + : 48u + ((e - 240u) % 4u); +} +uint tq1_0_digit_of(uint e) { + return e < 160u ? (e / 32u) + : e < 240u ? ((e - 160u) / 16u) + : ((e - 240u) / 4u); +} +// The 8-bit truncation below is part of the format, not an optimisation: +// the C reference does `uint8_t q = qs[..] * pow3[n]`. +uint tq1_0_trit(uint qbyte, uint t) { + const uint POW3_PACKED = (1u << 28) | (3u << 21) | (9u << 14) | (27u << 7) | 81u; + return ((((qbyte * ((POW3_PACKED >> (7u * (4u - t))) & 0x7Fu)) & 255u) * 3u) >> 8); +} + +#if defined(DATA_A_TQ1_0) +#define QUANT_K QUANT_K_TQ1_0 +#define QUANT_R 1 +#define A_TYPE block_tq1_0 +#define DATA_A_QUANT_K +#endif + #define QUANT_K_TQ2_0 256 // ternary (BitNet): 2-bit codes, w = (q - 1) * d; qs layout matches q2_K's @@ -949,6 +984,7 @@ shared uint16_t iq1s_grid[2048]; shared uint32_t iq1s_grid_gpu[2048]; #endif +#if defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -972,6 +1008,17 @@ void init_iq_shmem(uvec3 wgsize) barrier(); } #endif +#endif + +#if defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) +#if defined(DATA_A_IQ2_S) +shared uvec2 iq2s_grid[1024]; +#elif defined(DATA_A_IQ2_XS) +shared uvec2 iq2xs_grid[512]; +#else +shared uvec2 iq2xxs_grid[256]; +#endif +#endif #define QUANT_K_IQ2_XXS 256 #define QUANT_R_IQ2_XXS 1 @@ -1057,8 +1104,7 @@ const uvec2[256] iq2xxs_grid_const = { uvec2(0x08080808, 0x2b2b082b), uvec2(0x08192b08, 0x2b2b1908), uvec2(0x19190808, 0x2b2b2b08), uvec2(0x08081908, 0x2b2b2b19) }; -shared uvec2 iq2xxs_grid[256]; - +#if defined(DATA_A_IQ2_XXS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1070,12 +1116,15 @@ void init_iq_shmem(uvec3 wgsize) } barrier(); } +#endif +#if defined(DATA_A_IQ2_XXS) #define QUANT_K QUANT_K_IQ2_XXS #define QUANT_R QUANT_R_IQ2_XXS #define A_TYPE block_iq2_xxs #define A_TYPE_PACKED16 block_iq2_xxs_packed16 #endif +#endif #define QUANT_K_IQ2_XS 256 #define QUANT_R_IQ2_XS 1 @@ -1227,8 +1276,7 @@ const uvec2 iq2xs_grid_const[512] = { uvec2(0x082b2b08, 0x2b2b2b2b), uvec2(0x082b2b2b, 0x2b2b2b2b), uvec2(0x2b190819, 0x2b2b2b2b), uvec2(0x2b2b2b2b, 0x2b2b2b2b), }; -shared uvec2 iq2xs_grid[512]; - +#if defined(DATA_A_IQ2_XS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1240,12 +1288,15 @@ void init_iq_shmem(uvec3 wgsize) } barrier(); } +#endif +#if defined(DATA_A_IQ2_XS) #define QUANT_K QUANT_K_IQ2_XS #define QUANT_R QUANT_R_IQ2_XS #define A_TYPE block_iq2_xs #define A_TYPE_PACKED16 block_iq2_xs_packed16 #endif +#endif #define QUANT_K_IQ2_S 256 #define QUANT_R_IQ2_S 1 @@ -1527,8 +1578,7 @@ const uvec2 iq2s_grid_const[1024] = { uvec2(0x082b082b, 0x2b2b2b2b), uvec2(0x082b2b08, 0x2b2b2b2b), uvec2(0x2b082b08, 0x2b2b2b2b), uvec2(0x2b2b2b2b, 0x2b2b2b2b) }; -shared uvec2 iq2s_grid[1024]; - +#if defined(DATA_A_IQ2_S) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1540,12 +1590,23 @@ void init_iq_shmem(uvec3 wgsize) } barrier(); } +#endif +#if defined(DATA_A_IQ2_S) #define QUANT_K QUANT_K_IQ2_S #define QUANT_R QUANT_R_IQ2_S #define A_TYPE block_iq2_s #define A_TYPE_PACKED16 block_iq2_s_packed16 #endif +#endif + +#if defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) +#if defined(DATA_A_IQ3_S) +shared uint32_t iq3s_grid[512]; +#else +shared uint32_t iq3xxs_grid[256]; +#endif +#endif #define QUANT_K_IQ3_XXS 256 #define QUANT_R_IQ3_XXS 1 @@ -1599,8 +1660,7 @@ const uint32_t iq3xxs_grid_const[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; -shared uint32_t iq3xxs_grid[256]; - +#if defined(DATA_A_IQ3_XXS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1612,12 +1672,15 @@ void init_iq_shmem(uvec3 wgsize) } barrier(); } +#endif +#if defined(DATA_A_IQ3_XXS) #define QUANT_K QUANT_K_IQ3_XXS #define QUANT_R QUANT_R_IQ3_XXS #define A_TYPE block_iq3_xxs #define A_TYPE_PACKED16 block_iq3_xxs_packed16 #endif +#endif #define QUANT_K_IQ3_S 256 #define QUANT_R_IQ3_S 1 @@ -1709,8 +1772,7 @@ const uint32_t iq3s_grid_const[512] = { 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, }; -shared uint32_t iq3s_grid[512]; - +#if defined(DATA_A_IQ3_S) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1722,12 +1784,15 @@ void init_iq_shmem(uvec3 wgsize) } barrier(); } +#endif +#if defined(DATA_A_IQ3_S) #define QUANT_K QUANT_K_IQ3_S #define QUANT_R QUANT_R_IQ3_S #define A_TYPE block_iq3_s #define A_TYPE_PACKED16 block_iq3_s_packed16 #endif +#endif #define QUANT_K_IQ4_XS 256 #define QUANT_R_IQ4_XS 1 @@ -1839,17 +1904,38 @@ const int8_t kvalues_iq4nl_const[16] = { int8_t(1), int8_t(13), int8_t(25), int8_t(38), int8_t(53), int8_t(69), int8_t(89), int8_t(113) }; +#ifdef KVALUES_IQ4NL_I8 +shared int8_t kvalues_iq4nl[16]; +#else shared FLOAT_TYPE kvalues_iq4nl[16]; +#endif +#if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync for (uint i = gl_LocalInvocationIndex.x; i < kvalues_iq4nl.length(); i += wgsize.x) { +#ifdef KVALUES_IQ4NL_I8 + kvalues_iq4nl[i] = kvalues_iq4nl_const[i]; +#else kvalues_iq4nl[i] = FLOAT_TYPE(kvalues_iq4nl_const[i]); +#endif } barrier(); } + +#ifdef KVALUES_IQ4NL_I8 +i32vec2 iq4nl_to_i8x8(uint32_t vui) { + const u8vec4 i0 = unpack8( vui & 0x0F0F0F0F); + const u8vec4 i1 = unpack8((vui >> 4) & 0x0F0F0F0F); + + return i32vec2( + pack32(i8vec4(kvalues_iq4nl[i0.x], kvalues_iq4nl[i0.y], kvalues_iq4nl[i0.z], kvalues_iq4nl[i0.w])), + pack32(i8vec4(kvalues_iq4nl[i1.x], kvalues_iq4nl[i1.y], kvalues_iq4nl[i1.z], kvalues_iq4nl[i1.w]))); +} +#endif +#endif #endif #if defined(DATA_A_MXFP4) || defined(DATA_A_NVFP4) @@ -1895,7 +1981,7 @@ float ue4m3_to_fp32_build(uint u) { } #endif -#if !defined(USE_OCP_FP4) +#if (defined(DATA_A_MXFP4) || defined(DATA_A_NVFP4)) && !defined(USE_OCP_FP4) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp index 5ee5275d2782..9ee7769bab29 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp @@ -1,9 +1,23 @@ #version 450 #include "types.glsl" +#if defined(UNARY_MUL_FUSION) +#include "generic_binary_head.glsl" +#else #include "generic_unary_head.glsl" +#endif +#if defined(UNARY_MUL_FUSION) +// OP on src1 +layout(constant_id = 1) const bool op_on_b = false; +#endif + +#if defined(UNARY_MUL_FUSION) +layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; +const uint num_threads = 256; +#else layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; +#endif float op_abs(float x) { return abs(x); @@ -123,6 +137,7 @@ float op_gelu_erf(float a) { return 0.5f * a * (1.0f + sign_x * y); } +#if !defined(UNARY_MUL_FUSION) float op_xielu(float x) { const float alpha_n = p.param1; const float alpha_p = p.param2; @@ -136,6 +151,7 @@ float op_xielu(float x) { const float min_x_eps = min(x, eps); return (op_expm1(min_x_eps) - x) * alpha_n + beta * x; } +#endif float op_floor(float x) { return floor(x); @@ -155,8 +171,28 @@ float op_trunc(float x) { } void main() { - const uint idx = get_idx(); - + uint idx = get_idx(); + +#if defined(UNARY_MUL_FUSION) + // keep total threads at 512 + [[unroll]] for (uint iter = 0; iter < 2; ++iter) { + if (idx >= p.ne) { + continue; + } + uint i00, i01, i02, i03; + get_indices(idx, i00, i01, i02, i03); + + if (op_on_b) { + data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = + D_TYPE(FLOAT_TYPE(OP(float(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)])); + } else { + data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = + D_TYPE(FLOAT_TYPE(OP(float(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])); + } + + idx += num_threads; + } +#else if (idx >= p.ne) { return; } @@ -165,4 +201,5 @@ void main() { const uint d_idx = get_doffset() + dst_idx(idx); data_d[d_idx] = D_TYPE(OP(float(data_a[a_idx]))); +#endif } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index f55e4bb94442..aa000ac69514 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -72,6 +72,7 @@ const std::vector<std::string> type_names = { "iq4_nl", "mxfp4", "nvfp4", + "tq1_0", "tq2_0", "bf16", }; @@ -245,6 +246,17 @@ bool is_iq_quant(const std::string& type_name) { return string_starts_with(type_name, "iq"); } +bool is_lut_quant(const std::string& type_name) { + return is_iq_quant(type_name) || type_name == "mxfp4" || type_name == "nvfp4"; +} + +std::string lut_load_vec_a(const std::string& type_name) { + if (type_name == "iq1_s" || type_name == "iq1_m" || type_name == "iq2_xxs" || type_name == "iq2_xs" || type_name == "iq2_s" || type_name == "iq4_xs") { + return "8"; + } + return "4"; +} + static const char path_separator = '/'; std::string join_paths(const std::string& path1, const std::string& path2) { @@ -582,20 +594,28 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c } for (const auto& tname : type_names) { - std::string load_vec_quant = "2"; - if ((tname == "q1_0") || (tname == "q4_0") || (tname == "q4_1") || (tname == "q5_1") || (tname == "iq1_s") || (tname == "iq1_m") || (tname == "iq2_xxs") || (tname == "iq2_xs") || (tname == "iq2_s")) - load_vec_quant = "8"; - else if ((tname == "q2_0") || (tname == "q5_0") || (tname == "q8_0") || (tname == "q2_k") || (tname == "q4_k") || (tname == "q5_k") || (tname == "iq3_xxs") || (tname == "iq3_s") || (tname == "iq4_xs") || (tname == "iq4_nl") || (tname == "mxfp4") || (tname == "nvfp4")) - load_vec_quant = "4"; - if (tname == "bf16") { continue; } - std::string data_a_key = "DATA_A_" + to_uppercase(tname); - // For aligned matmul loads - std::string load_vec_a = (coopmat2 || tname == "f32" || tname == "f16" || tname == "bf16") ? load_vec : load_vec_quant; + // Float types keep per-type compilation (different accumulation loop structure) + if (tname == "f32" || tname == "f16") { + std::string data_a_key = "DATA_A_" + to_uppercase(tname); + + const std::map<std::string, std::string> float_type_dict = { + {"FLOAT_TYPE", FLOAT_TYPE(1, tname)}, + {"FLOAT_TYPEV2", FLOAT_TYPE(2, tname)}, + {"FLOAT_TYPEV4", FLOAT_TYPE(4, tname)}, + {"FLOAT_TYPEV8", FLOAT_TYPE(8, tname)}, + }; + + if (!coopmat2) { + string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } + continue; + } + std::string data_a_key = "DATA_A_" + to_uppercase(tname); const std::map<std::string, std::string> float_type_dict = { {"FLOAT_TYPE", FLOAT_TYPE(1, tname)}, {"FLOAT_TYPEV2", FLOAT_TYPE(2, tname)}, @@ -603,30 +623,52 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c {"FLOAT_TYPEV8", FLOAT_TYPE(8, tname)}, }; - // don't generate f32 variants for coopmat2 - if (!coopmat2) { - string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (!f16acc && !coopmat && !coopmat2 && !dot2 && (is_legacy_quant(tname) || is_k_quant(tname) || tname == "mxfp4" || tname == "iq3_s" || tname == "iq4_xs")) { + string_to_spv(shader_name + "_" + tname + "_q8_1", "mul_mmq.comp", merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"D_TYPE", "float"},}), fp16, coopmat, coopmat2, f16acc); } +#endif - if (tname != "f16" && tname != "f32") { - string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); - } + if (is_lut_quant(tname)) { + std::string lva = lut_load_vec_a(tname); + + string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if ((coopmat || coopmat2) && (tname == "mxfp4" || tname == "nvfp4")) { if (!coopmat2) { - string_to_spv(shader_name + "_" + tname + "_f32_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } + +#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) + if ((tname == "mxfp4" || tname == "nvfp4") && (coopmat || coopmat2)) { + string_to_spv(shader_name + "_" + tname + "_f16_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + if (!coopmat2) { + string_to_spv(shader_name + "_" + tname + "_f32_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } } - string_to_spv(shader_name + "_" + tname + "_f16_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); - } #endif + continue; + } -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - // Integer dot mmq performs better with f32 accumulators (different shader, skip for dot2) - if (!f16acc && !coopmat && !coopmat2 && !dot2 && (is_legacy_quant(tname) || is_k_quant(tname) || tname == "mxfp4")) { - string_to_spv(shader_name + "_" + tname + "_q8_1", "mul_mmq.comp", merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"D_TYPE", "float"},}), fp16, coopmat, coopmat2, f16acc); + // dedicated shader needed due to regression on Ampere + if (coopmat2 && (tname == "q4_k" || tname == "q5_k")) { + string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } + } + + // Quant shader: one SPIR-V for all quant types, selected via MmTypeA spec constant + { + const std::map<std::string, std::string> quant_float_type_dict = { + {"FLOAT_TYPE", FLOAT_TYPE(1, "q4_0")}, + {"FLOAT_TYPEV2", FLOAT_TYPE(2, "q4_0")}, + {"FLOAT_TYPEV4", FLOAT_TYPE(4, "q4_0")}, + {"FLOAT_TYPEV8", FLOAT_TYPE(8, "q4_0")}, + }; + + string_to_spv(shader_name + "_quant_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + + if (!coopmat2) { + string_to_spv(shader_name + "_quant_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); } -#endif } } @@ -734,7 +776,7 @@ void process_shaders() { for (const auto& tname : type_names) { // mul mat vec std::string data_a_key = "DATA_A_" + to_uppercase(tname); - std::string shader = (string_ends_with(tname, "_k") || string_starts_with(tname, "iq1_") || string_starts_with(tname, "iq2_") || string_starts_with(tname, "iq3_") || tname == "tq2_0") ? "mul_mat_vec_" + tname + ".comp" : "mul_mat_vec.comp"; + std::string shader = (string_ends_with(tname, "_k") || string_starts_with(tname, "iq1_") || string_starts_with(tname, "iq2_") || string_starts_with(tname, "iq3_") || tname == "iq4_xs" || tname == "tq2_0" || tname == "tq1_0") ? "mul_mat_vec_" + tname + ".comp" : "mul_mat_vec.comp"; string_to_spv("mul_mat_vec_" + tname + "_f32_f32", shader, merge_maps(base_dict, {{data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}})); string_to_spv("mul_mat_vec_" + tname + "_f16_f32", shader, merge_maps(base_dict, {{data_a_key, "1"}, {"B_TYPE", "float16_t"}, {"B_TYPEV2", "f16vec2"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}})); @@ -765,7 +807,7 @@ void process_shaders() { // mul mat vec with integer dot product #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (is_legacy_quant(tname) || tname == "mxfp4" || is_k_quant(tname) || tname == "iq1_s" || tname == "iq1_m") { + if (is_legacy_quant(tname) || tname == "mxfp4" || is_k_quant(tname) || tname == "iq1_s" || tname == "iq1_m" || tname == "iq4_xs") { string_to_spv("mul_mat_vec_" + tname + "_q8_1_f32", "mul_mat_vecq.comp", merge_maps(base_dict, {{data_a_key, "1"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"FLOAT_TYPEV2", "vec2"}, {"ACC_TYPE", "float"}})); string_to_spv("mul_mat_vec_" + tname + "_q8_1_f32_subgroup", "mul_mat_vecq.comp", merge_maps(base_dict, {{data_a_key, "1"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"FLOAT_TYPEV2", "vec2"}, {"ACC_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}})); string_to_spv("mul_mat_vec_" + tname + "_q8_1_f32_subgroup_no_shmem", "mul_mat_vecq.comp", merge_maps(base_dict, {{data_a_key, "1"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"FLOAT_TYPEV2", "vec2"}, {"ACC_TYPE", "float"}, {"USE_SUBGROUP_ADD_NO_SHMEM", "1"}})); @@ -805,6 +847,10 @@ void process_shaders() { string_to_spv("norm_f32", "norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rms_norm_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); + string_to_spv("rms_norm_mul_add_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_ADD_FUSION", "1"}})); + string_to_spv("rms_norm_mul_add_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_ADD_FUSION", "1"}})); + string_to_spv("rms_norm_set_rows_f32_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}})); + string_to_spv("rms_norm_set_rows_f32_f16", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}})); string_to_spv("rms_norm_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rms_norm_mul_rope_f32_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}})); string_to_spv("rms_norm_mul_rope_f32_f16", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"ROPE_D_TYPE", "float16_t"}, {"RMS_NORM_ROPE_FUSION", "1"}})); @@ -930,6 +976,12 @@ void process_shaders() { string_to_spv("fa_mask_opt", "flash_attn_mask_opt.comp", {}); + string_to_spv("fa_decode_ph1", "flash_attn_decode_phase_1.comp", {}, true, true, false, false); + string_to_spv("fa_decode_ph2", "flash_attn_decode_phase_2.comp", {}, true, true, false, false); + + string_to_spv("fa_sparse_compact", "flash_attn_sparse_compact.comp", {}); + string_to_spv("fa_sparse_compact_subgroup", "flash_attn_sparse_compact.comp", {{"USE_SUBGROUPS", "1"}}); + string_to_spv("quantize_q8_1", "quantize_q8_1.comp", {}); string_to_spv("quantize_q8_1_subgroup", "quantize_q8_1.comp", {{"USE_SUBGROUPS", "1"}}); @@ -1014,6 +1066,15 @@ void process_shaders() { string_to_spv("softplus_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_softplus"}}); string_to_spv("softplus_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_softplus"}}); + string_to_spv("gelu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("gelu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("sigmoid_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("sigmoid_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("silu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("silu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("softplus_mul_f32","unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("softplus_mul_f16","unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("add1_f16_f16", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}}); string_to_spv("add1_f16_f32", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}}); string_to_spv("add1_f32_f32", "add1.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); @@ -1090,6 +1151,9 @@ void process_shaders() { string_to_spv("fwht_f32", "fwht.comp", {}); string_to_spv("fwht_shmem_f32", "fwht.comp", {{"FWHT_SHMEM", "1"}}); string_to_spv("count_equal_i32", "count_equal.comp", merge_maps(base_dict, {{"A_TYPE", "int"}, {"B_TYPE", "int"}, {"D_TYPE", "int"}})); + string_to_spv("dsv4_hc_comb_f32", "dsv4_hc_comb.comp", {}); + string_to_spv("dsv4_hc_pre_f32", "dsv4_hc_pre.comp", {}); + string_to_spv("dsv4_hc_post_f32", "dsv4_hc_post.comp", {}); string_to_spv("cumsum_f32", "cumsum.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("cumsum_multipass1_f32", "cumsum_multipass1.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("cumsum_multipass2_f32", "cumsum_multipass2.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); @@ -1335,7 +1399,7 @@ void write_output_files() { for (const std::string& btype : btypes) { for (const auto& tname : type_names) { - if (btype == "q8_1" && !is_legacy_quant(tname) && tname != "mxfp4" && !is_k_quant(tname) && tname != "iq1_s" && tname != "iq1_m") { + if (btype == "q8_1" && !is_legacy_quant(tname) && tname != "mxfp4" && !is_k_quant(tname) && tname != "iq1_s" && tname != "iq1_m" && tname != "iq4_xs") { continue; } hdr << "extern const void * arr_dmmv_" << tname << "_" << btype << "_f32_data[3];\n"; diff --git a/ggml/src/ggml-webgpu/CMakeLists.txt b/ggml/src/ggml-webgpu/CMakeLists.txt index 1503a1ef8ba6..2eacca7f2b3b 100644 --- a/ggml/src/ggml-webgpu/CMakeLists.txt +++ b/ggml/src/ggml-webgpu/CMakeLists.txt @@ -39,6 +39,12 @@ ggml_add_backend_library(ggml-webgpu add_dependencies(ggml-webgpu generate_shaders) +# Dawn needs C++20 (https://dawn.googlesource.com/dawn/+/refs/heads/main/docs/quickstart-cmake.md#prerequisites) +target_compile_features(ggml-webgpu PRIVATE cxx_std_20) + +# Disable C++20 module scanning since emscan-deps fails to find webgpu_cpp.h +set_target_properties(ggml-webgpu PROPERTIES CXX_SCAN_FOR_MODULES OFF) + if(EMSCRIPTEN) set(EMDAWNWEBGPU_DIR "" CACHE PATH "Path to emdawnwebgpu_pkg") diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp index a7ff36030fab..47a266d7de3e 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp @@ -81,6 +81,7 @@ struct ggml_webgpu_shader_lib_context { ggml_tensor * src4; ggml_tensor * src5; ggml_tensor * dst; + ggml_tensor * dst_fuse; uint32_t max_wg_size; size_t wg_mem_limit_bytes = 0; @@ -106,6 +107,11 @@ struct ggml_webgpu_generic_shader_decisions { bool inplace = false; }; +struct ggml_webgpu_get_rows_shader_decisions { + uint32_t wg_size = 0; + bool vectorized = false; +}; + struct ggml_webgpu_binary_shader_decisions { uint32_t wg_size = 0; bool inplace = false; @@ -407,12 +413,13 @@ struct ggml_webgpu_im2col_pipeline_key_hash { /** Gated Delta Net **/ struct ggml_webgpu_gated_delta_net_pipeline_key { - int type; - int s_v; - int kda; + int type; + int s_v; + int kda; + bool fused_cache; bool operator==(const ggml_webgpu_gated_delta_net_pipeline_key & other) const { - return type == other.type && s_v == other.s_v && kda == other.kda; + return type == other.type && s_v == other.s_v && kda == other.kda && fused_cache == other.fused_cache; } }; @@ -1551,8 +1558,8 @@ class ggml_webgpu_shader_lib { return argsort_merge_pipelines[order]; } - webgpu_pipeline get_get_rows_pipeline(const ggml_webgpu_shader_lib_context & context) { - const bool vectorized = context.src0->type == GGML_TYPE_F32 && context.dst->ne[0] % 4 == 0; + webgpu_pipeline get_get_rows_pipeline(const ggml_webgpu_shader_lib_context & context, bool vec4_aligned) { + const bool vectorized = context.src0->type == GGML_TYPE_F32 && context.dst->ne[0] % 4 == 0 && vec4_aligned; ggml_webgpu_get_rows_pipeline_key key = {}; key.src_type = context.src0->type; key.vectorized = (int) vectorized; @@ -1669,8 +1676,9 @@ class ggml_webgpu_shader_lib { defines.push_back("WG_SIZE=" + std::to_string(context.max_wg_size)); auto processed = preprocessor.preprocess(wgsl_get_rows, defines); - auto decisions = std::make_shared<ggml_webgpu_generic_shader_decisions>(); + auto decisions = std::make_shared<ggml_webgpu_get_rows_shader_decisions>(); decisions->wg_size = context.max_wg_size; + decisions->vectorized = vectorized; webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, variant); pipeline.context = decisions; get_rows_pipelines[key] = pipeline; @@ -1859,6 +1867,7 @@ class ggml_webgpu_shader_lib { key.type = context.dst->type; key.s_v = (int) context.src2->ne[0]; key.kda = context.src3->ne[0] == context.src2->ne[0]; + key.fused_cache = context.dst_fuse != nullptr; auto it = gated_delta_net_pipelines.find(key); if (it != gated_delta_net_pipelines.end()) { @@ -1881,6 +1890,11 @@ class ggml_webgpu_shader_lib { variant += "_kda"; } + if (key.fused_cache) { + defines.push_back("FUSED_CACHE"); + variant += "_fused_cache"; + } + defines.push_back("S_V=" + std::to_string(key.s_v) + "u"); defines.push_back("WG_SIZE=" + std::to_string(key.s_v) + "u"); diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 1a43c72733bc..9c5dc768efe7 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -374,20 +374,28 @@ static wgpu::Buffer ggml_webgpu_tensor_buf(const ggml_tensor * tensor) { return ctx->buffer; } +// Binding offset for a tensor: the largest aligned offset at or before the tensor whose +// distance to the tensor is a whole number of type blocks, so shaders can index the +// misalignment in elements even for block quantized types. +static size_t ggml_webgpu_tensor_align_offset(const ggml_tensor * t, size_t alignment) { + const size_t offset = ggml_webgpu_tensor_offset(t); + const size_t type_size = ggml_type_size(t->type); + size_t aligned = offset & ~(alignment - 1); + while ((offset - aligned) % type_size != 0) { + GGML_ASSERT(aligned >= alignment); + aligned -= alignment; + } + return aligned; +} + static size_t ggml_webgpu_tensor_misalignment(const ggml_tensor * t, size_t alignment) { - size_t offset = ggml_webgpu_tensor_offset(t); - return offset & (alignment - 1); + return ggml_webgpu_tensor_offset(t) - ggml_webgpu_tensor_align_offset(t, alignment); } static size_t ggml_webgpu_tensor_misalignment(webgpu_context & ctx, const ggml_tensor * t) { return ggml_webgpu_tensor_misalignment(t, ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); } -static size_t ggml_webgpu_tensor_align_offset(const ggml_tensor * t, size_t alignment) { - size_t offset = ggml_webgpu_tensor_offset(t); - return offset & ~(alignment - 1); -} - static size_t ggml_webgpu_tensor_align_offset(webgpu_context & ctx, const ggml_tensor * t) { return ggml_webgpu_tensor_align_offset(t, ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); } @@ -1375,7 +1383,8 @@ static webgpu_encoded_op ggml_webgpu_gated_delta_net(webgpu_context & ctx, ggml_tensor * src3, ggml_tensor * src4, ggml_tensor * src5, - ggml_tensor * dst) { + ggml_tensor * dst, + ggml_tensor * dst_fuse) { ggml_webgpu_shader_lib_context shader_lib_ctx = {}; shader_lib_ctx.src0 = src0; shader_lib_ctx.src1 = src1; @@ -1383,6 +1392,7 @@ static webgpu_encoded_op ggml_webgpu_gated_delta_net(webgpu_context & ctx, shader_lib_ctx.src3 = src3; shader_lib_ctx.src4 = src4; shader_lib_ctx.dst = dst; + shader_lib_ctx.dst_fuse = dst_fuse; shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; webgpu_pipeline pipeline = ctx->shader_lib->get_gated_delta_net_pipeline(shader_lib_ctx); @@ -1418,6 +1428,8 @@ static webgpu_encoded_op ggml_webgpu_gated_delta_net(webgpu_context & ctx, (uint32_t) (src2->ne[3] / src0->ne[3]), K, scale_u32, + dst_fuse ? (uint32_t) (dst_fuse->nb[2] / ggml_type_size(dst_fuse->type)) : 0, + dst_fuse ? (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst_fuse) / ggml_type_size(dst_fuse->type)) : 0, }; std::vector<wgpu::BindGroupEntry> entries = { @@ -1427,6 +1439,10 @@ static webgpu_encoded_op ggml_webgpu_gated_delta_net(webgpu_context & ctx, ggml_webgpu_make_tensor_bind_group_entry(ctx, 6, dst), }; + if (dst_fuse) { + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 7, dst_fuse)); + } + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, h, n_seqs); } @@ -1510,15 +1526,24 @@ static webgpu_encoded_op ggml_webgpu_get_rows(webgpu_context & ctx, shader_lib_ctx.dst = dst; shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; - webgpu_pipeline pipeline = ctx->shader_lib->get_get_rows_pipeline(shader_lib_ctx); - auto * decisions = static_cast<ggml_webgpu_generic_shader_decisions *>(pipeline.context.get()); + const uint32_t offset_src = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src) / ggml_type_size(src->type)); + const uint32_t offset_dst = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)); + const uint32_t stride_src1 = (uint32_t) (src->nb[1] / ggml_type_size(src->type)); + const uint32_t stride_src2 = (uint32_t) (src->nb[2] / ggml_type_size(src->type)); + const uint32_t stride_src3 = (uint32_t) (src->nb[3] / ggml_type_size(src->type)); - std::vector<uint32_t> params = { (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src) / ggml_type_size(src->type)), + const bool vec4_aligned = offset_src % 4 == 0 && offset_dst % 4 == 0 && stride_src1 % 4 == 0 && + stride_src2 % 4 == 0 && stride_src3 % 4 == 0; + + webgpu_pipeline pipeline = ctx->shader_lib->get_get_rows_pipeline(shader_lib_ctx, vec4_aligned); + auto * decisions = static_cast<ggml_webgpu_get_rows_shader_decisions *>(pipeline.context.get()); + + std::vector<uint32_t> params = { offset_src, (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, idx) / ggml_type_size(idx->type)), - (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)), - (uint32_t) (src->nb[1] / ggml_type_size(src->type)), - (uint32_t) (src->nb[2] / ggml_type_size(src->type)), - (uint32_t) (src->nb[3] / ggml_type_size(src->type)), + offset_dst, + stride_src1, + stride_src2, + stride_src3, (uint32_t) (idx->nb[0] / ggml_type_size(idx->type)), (uint32_t) (idx->nb[1] / ggml_type_size(idx->type)), (uint32_t) (idx->nb[2] / ggml_type_size(idx->type)), @@ -1536,7 +1561,7 @@ static webgpu_encoded_op ggml_webgpu_get_rows(webgpu_context & ctx, ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, idx), ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, dst) }; - uint32_t blocks_per_row = (uint32_t) (dst->ne[0] / (src->type == GGML_TYPE_F32 && dst->ne[0] % 4 == 0 ? 4 : 1)); + uint32_t blocks_per_row = (uint32_t) (dst->ne[0] / (decisions->vectorized ? 4 : 1)); uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); uint32_t total_threads = float_parallel ? blocks_per_row * total_rows : total_rows; uint32_t wg_x = CEIL_DIV(total_threads, decisions->wg_size); @@ -3203,6 +3228,67 @@ static bool ggml_webgpu_can_fuse_rms_norm_mul(const struct ggml_cgraph * cgraph, return true; } +static bool ggml_webgpu_can_fuse_gdn_cache(const struct ggml_cgraph * cgraph, int node_idx, int & num_encoded_ops) { + const ggml_tensor * gdn = cgraph->nodes[node_idx]; + + // the kernel skips the snapshot tail, so the gdn output must not be a graph output + if (gdn->op != GGML_OP_GATED_DELTA_NET || gdn->type != GGML_TYPE_F32 || (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + + const ggml_tensor * src_v = gdn->src[2]; + const int64_t S_v = src_v->ne[0]; + const int64_t H = src_v->ne[1]; + const int64_t n_tokens = src_v->ne[2]; + const int64_t n_seqs = src_v->ne[3]; + const int64_t D = S_v * S_v * H; + const int64_t K = ggml_get_op_params_i32(gdn, 0); // snapshot slot count + const int64_t n_written = std::min<int64_t>(n_tokens, K); // newest n_written slots are written + + // snapshot tail starts right after the attention scores + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs); + + // snapshot cpy is the first real node after the gdn (skip views/no-ops) + const ggml_tensor * cpy = nullptr; + int cpy_idx = 0; + for (int j = node_idx + 1; j < cgraph->n_nodes && cpy == nullptr; ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + if (ggml_op_is_empty(n->op) || ggml_is_empty(n)) { + continue; + } + if (n->op != GGML_OP_CPY || (n->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + cpy = n; + cpy_idx = j; + } + if (cpy == nullptr) { + return false; + } + + const ggml_tensor * cpy_src = cpy->src[0]; // view of the gdn snapshot tail + const ggml_tensor * cpy_dst = cpy->src[1]; // cache view the kernel writes to + + // src must be this gdn's snapshot tail (contiguous, at the tail offset) + if (cpy_src->op != GGML_OP_VIEW || cpy_src->view_src != gdn || cpy_src->view_offs != tail_off || + !ggml_is_contiguous(cpy_src)) { + return false; + } + + // dst is the [D, n_seqs, n_written] cache view; require nb[1] == D (the per-seq stride the kernel + // assumes). ggml_cpy pins src to the same element count. + const std::array<int64_t, GGML_MAX_DIMS> expected_ne = { D, n_seqs, n_written, 1 }; + if (cpy_dst->op != GGML_OP_VIEW || cpy_dst->type != GGML_TYPE_F32 || cpy_dst->data == nullptr || + !std::equal(expected_ne.begin(), expected_ne.end(), cpy_dst->ne) || + cpy_dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || cpy_dst->nb[1] != (size_t) ggml_row_size(GGML_TYPE_F32, D)) { + return false; + } + + num_encoded_ops = cpy_idx - node_idx + 1; + + return true; +} + static webgpu_encoded_op ggml_webgpu_upscale(webgpu_context ctx, ggml_tensor * src, ggml_tensor * dst) { const uint32_t mode_flags = (uint32_t) ggml_get_op_params_i32(dst, 0); std::vector<uint32_t> params = { (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src) / ggml_type_size(src->type)), @@ -3341,7 +3427,14 @@ static std::optional<webgpu_encoded_op> ggml_webgpu_encode(webgpu_context ctx, return ggml_webgpu_ssm_scan(ctx, src0, src1, src2, node->src[3], node->src[4], node->src[5], node->src[6], node); case GGML_OP_GATED_DELTA_NET: - return ggml_webgpu_gated_delta_net(ctx, src0, src1, src2, node->src[3], node->src[4], node->src[5], node); + if (ggml_webgpu_can_fuse_gdn_cache(cgraph, node_idx, num_encoded_ops)) { + ggml_tensor * dst_fuse = cgraph->nodes[node_idx + num_encoded_ops - 1]->src[1]; + return ggml_webgpu_gated_delta_net(ctx, src0, src1, src2, node->src[3], node->src[4], node->src[5], + node, dst_fuse); + } else { + return ggml_webgpu_gated_delta_net(ctx, src0, src1, src2, node->src[3], node->src[4], node->src[5], + node, nullptr); + } case GGML_OP_PAD: return ggml_webgpu_pad(ctx, src0, node); case GGML_OP_ARGMAX: @@ -4006,16 +4099,17 @@ static void ggml_backend_webgpu_request_adapter(wgpu::Instance & instance, wgpu: options.nextInChain = &adapterTogglesDesc; #endif - instance.WaitAny(instance.RequestAdapter( - &options, wgpu::CallbackMode::AllowSpontaneous, - [&adapter](wgpu::RequestAdapterStatus status, wgpu::Adapter _adapter, const char * message) { - if (status != wgpu::RequestAdapterStatus::Success) { - GGML_LOG_ERROR("ggml_webgpu: Failed to get an adapter: %s\n", message); - return; - } - adapter = std::move(_adapter); - }), - UINT64_MAX); + instance.WaitAny( + instance.RequestAdapter( + &options, wgpu::CallbackMode::AllowSpontaneous, + [&adapter](wgpu::RequestAdapterStatus status, wgpu::Adapter _adapter, wgpu::StringView message) { + if (status != wgpu::RequestAdapterStatus::Success) { + GGML_LOG_ERROR("ggml_webgpu: Failed to get an adapter: %s\n", std::string(message).c_str()); + return; + } + adapter = std::move(_adapter); + }), + UINT64_MAX); } static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { @@ -4412,6 +4506,9 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const default: break; } + if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { + supports_op = false; + } break; case GGML_OP_FLASH_ATTN_EXT: { diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl index 7d7b34755493..6f4b5a31c16c 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl @@ -19,6 +19,16 @@ var<storage, read_write> src_state: array<f32>; @group(0) @binding(6) var<storage, read_write> dst: array<f32>; +#ifdef FUSED_CACHE +@group(0) @binding(7) +var<storage, read_write> dst_fuse: array<f32>; +#define DST_SNAP dst_fuse +#define PARAMS_BINDING 8 +#else +#define DST_SNAP dst +#define PARAMS_BINDING 7 +#endif + struct Params { h: u32, n_tokens: u32, @@ -41,9 +51,11 @@ struct Params { rq3: u32, K: u32, scale: f32, + dst_fuse_nb2: u32, + dst_fuse_off: u32, }; -@group(0) @binding(7) +@group(0) @binding(PARAMS_BINDING) var<uniform> params: Params; var<workgroup> sh_k: array<f32, S_V>; @@ -66,7 +78,14 @@ fn main( // input state holds s0 only [S_v, S_v, H, n_seqs]: per-seq stride is H*D. let state_in_base = (seq_id * params.h + head_id) * state_size; let state_out_base = (seq_id * params.h + head_id) * state_size; + +#ifdef FUSED_CACHE + let state_size_per_snap = params.dst_fuse_nb2; + let snap_off = params.dst_fuse_off; +#else let state_size_per_snap = state_size * params.h * params.n_seqs; + let snap_off = params.s_off; +#endif var state: array<f32, S_V>; for (var i = 0u; i < S_V; i++) { @@ -131,9 +150,9 @@ fn main( // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back. let target_slot = i32(params.n_tokens) - 1 - i32(t); if (target_slot >= 0 && target_slot < i32(params.K)) { - let slot_base = params.s_off + u32(target_slot) * state_size_per_snap + state_out_base; + let slot_base = snap_off + u32(target_slot) * state_size_per_snap + state_out_base; for (var i = 0u; i < S_V; i++) { - dst[slot_base + col * S_V + i] = state[i]; + DST_SNAP[slot_base + col * S_V + i] = state[i]; } } } @@ -143,7 +162,7 @@ fn main( if (params.K == 1u) { for (var i = 0u; i < S_V; i++) { - dst[params.s_off + state_out_base + col * S_V + i] = state[i]; + DST_SNAP[snap_off + state_out_base + col * S_V + i] = state[i]; } } } diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index d5daacbb0c3a..a4464ee44cff 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -3485,6 +3485,57 @@ struct ggml_tensor * ggml_sinkhorn_norm( return result; } +// ggml_prec + +bool ggml_prec_set_acc( + struct ggml_tensor * a, + enum ggml_prec prec) { + switch (a->op) { + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + { + const int32_t prec_i32 = (int32_t) prec; + ggml_set_op_params_i32(a, 0, prec_i32); + } + break; + case GGML_OP_FLASH_ATTN_EXT: + { + const int32_t prec_i32 = (int32_t) prec; + ggml_set_op_params_i32(a, 3, prec_i32); + } + break; + default: + return false; + }; + + return true; +} + +bool ggml_prec_set_src( + struct ggml_tensor * a, + enum ggml_prec prec, + int idx) { + GGML_ASSERT(idx >= 0 && idx < GGML_MAX_SRC); + + switch (a->op) { + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + { + if (idx != 1) { + return false; + } + + const int32_t prec_i32 = (int32_t) prec; + ggml_set_op_params_i32(a, 2 + idx, prec_i32); + } + break; + default: + return false; + }; + + return true; +} + // ggml_mul_mat static inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { @@ -4093,8 +4144,8 @@ struct ggml_tensor * ggml_permute( struct ggml_tensor * result = ggml_view_tensor(ctx, a); ggml_format_name(result, "%s (permuted)", a->name); - int ne[GGML_MAX_DIMS]; - int nb[GGML_MAX_DIMS]; + int64_t ne[GGML_MAX_DIMS]; + size_t nb[GGML_MAX_DIMS]; ne[axis0] = a->ne[0]; ne[axis1] = a->ne[1]; @@ -6843,10 +6894,12 @@ struct ggml_tensor * ggml_dsv4_hc_comb( // ggml_dsv4_hc_pre -struct ggml_tensor * ggml_dsv4_hc_pre( +static struct ggml_tensor * ggml_dsv4_hc_pre_impl( struct ggml_context * ctx, struct ggml_tensor * x, - struct ggml_tensor * weights) { + struct ggml_tensor * weights, + float scale, + bool gated) { GGML_ASSERT(x->type == GGML_TYPE_F32); GGML_ASSERT(weights->type == GGML_TYPE_F32); @@ -6856,13 +6909,22 @@ struct ggml_tensor * ggml_dsv4_hc_pre( GGML_ASSERT(hc > 0); GGML_ASSERT(x->ne[3] == 1); - GGML_ASSERT(weights->ne[0] == hc); - GGML_ASSERT(weights->ne[1] == n_tokens); - GGML_ASSERT(weights->ne[2] == 1); + if (gated) { + GGML_ASSERT(weights->ne[0] == n_embd); + GGML_ASSERT(weights->ne[1] == hc); + GGML_ASSERT(weights->ne[2] == n_tokens); + } else { + GGML_ASSERT(weights->ne[0] == hc); + GGML_ASSERT(weights->ne[1] == n_tokens); + GGML_ASSERT(weights->ne[2] == 1); + } GGML_ASSERT(weights->ne[3] == 1); struct ggml_tensor * result = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens); + ggml_set_op_params_f32(result, 0, scale); + ggml_set_op_params_i32(result, 1, gated ? 1 : 0); + result->op = GGML_OP_DSV4_HC_PRE; result->src[0] = x; result->src[1] = weights; @@ -6870,6 +6932,21 @@ struct ggml_tensor * ggml_dsv4_hc_pre( return result; } +struct ggml_tensor * ggml_dsv4_hc_pre( + struct ggml_context * ctx, + struct ggml_tensor * x, + struct ggml_tensor * weights) { + return ggml_dsv4_hc_pre_impl(ctx, x, weights, 1.0f, false); +} + +struct ggml_tensor * ggml_dsv4_hc_pre_gated( + struct ggml_context * ctx, + struct ggml_tensor * x, + struct ggml_tensor * gate, + float scale) { + return ggml_dsv4_hc_pre_impl(ctx, x, gate, scale, true); +} + // ggml_dsv4_hc_post struct ggml_tensor * ggml_dsv4_hc_post( @@ -6881,7 +6958,6 @@ struct ggml_tensor * ggml_dsv4_hc_post( GGML_ASSERT(x->type == GGML_TYPE_F32); GGML_ASSERT(residual->type == GGML_TYPE_F32); GGML_ASSERT(post->type == GGML_TYPE_F32); - GGML_ASSERT(comb->type == GGML_TYPE_F32); const int64_t n_embd = x->ne[0]; const int64_t n_tokens = x->ne[1]; @@ -6900,10 +6976,13 @@ struct ggml_tensor * ggml_dsv4_hc_post( GGML_ASSERT(post->ne[2] == 1); GGML_ASSERT(post->ne[3] == 1); - GGML_ASSERT(comb->ne[0] == hc); - GGML_ASSERT(comb->ne[1] == hc); - GGML_ASSERT(comb->ne[2] == n_tokens); - GGML_ASSERT(comb->ne[3] == 1); + if (comb) { + GGML_ASSERT(comb->type == GGML_TYPE_F32); + GGML_ASSERT(comb->ne[0] == hc); + GGML_ASSERT(comb->ne[1] == hc); + GGML_ASSERT(comb->ne[2] == n_tokens); + GGML_ASSERT(comb->ne[3] == 1); + } struct ggml_tensor * result = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd, hc, n_tokens); diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index 6c7b5817812b..0eb9fb744d28 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -9,6 +9,7 @@ #include <cstdio> #include <cstdlib> #include <cstring> +#include <cerrno> #include <map> #include <new> #include <stdexcept> @@ -237,6 +238,7 @@ struct gguf_reader { : callback(callback), userdata(userdata), max_chunk_read(max_chunk_read), + start_offset(data_offset), data_offset(data_offset), nbytes_remain(nbytes_remain) { GGML_ASSERT(max_chunk_read > 0); @@ -365,6 +367,11 @@ struct gguf_reader { return data_offset; } + // position in the file where the GGUF data starts, alignment is relative to it, not to the file + uint64_t start() const { + return start_offset; + } + bool seek(uint64_t absolute_offset) const { const uint64_t end_offset = uint64_t(data_offset) + nbytes_remain; if (absolute_offset > end_offset) { @@ -414,6 +421,7 @@ struct gguf_reader { gguf_reader_callback_t callback = nullptr; void * userdata = nullptr; size_t max_chunk_read = 0; + uint64_t start_offset = 0; mutable uint64_t data_offset = 0; mutable uint64_t nbytes_remain = 0; }; @@ -762,7 +770,7 @@ static struct gguf_context * gguf_init_from_reader(const struct gguf_reader & gr GGML_ASSERT(int64_t(ctx->info.size()) == n_tensors); // we require the data section to be aligned, so take into account any padding - if (n_tensors > 0 && !gr.seek(GGML_PAD(gr.tell(), ctx->alignment))) { + if (n_tensors > 0 && !gr.seek(gr.start() + GGML_PAD(gr.tell() - gr.start(), ctx->alignment))) { GGML_LOG_ERROR("%s: failed to seek to beginning of data section\n", __func__); gguf_free(ctx); return nullptr; diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 7d2a2d02111c..e010019bfe27 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -231,6 +231,7 @@ class Attention: KV_LORA_RANK_SWA = "{arch}.attention.kv_lora_rank_swa" SHARED_KV_LAYERS = "{arch}.attention.shared_kv_layers" SLIDING_WINDOW_PATTERN = "{arch}.attention.sliding_window_pattern" + RECURRENT_LAYERS = "{arch}.attention.recurrent_layers" TEMPERATURE_SCALE = "{arch}.attention.temperature_scale" ROPE_PATTERN = "{arch}.attention.rope_pattern" @@ -292,6 +293,12 @@ class Split: LLM_KV_SPLIT_COUNT = "split.count" LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count" + class HRM: + LAYERS_PER_STACK = "{arch}.hrm.layers_per_stack" + H_CYCLES = "{arch}.hrm.h_cycles" + L_CYCLES = "{arch}.hrm.l_cycles" + PREFIX_LM = "{arch}.hrm.prefix_lm" + class SSM: CONV_KERNEL = "{arch}.ssm.conv_kernel" INNER_SIZE = "{arch}.ssm.inner_size" @@ -526,6 +533,7 @@ class MODEL_ARCH(IntEnum): QWEN3 = auto() QWEN3MOE = auto() QWEN3NEXT = auto() + HRM_TEXT = auto() QWEN3VL = auto() QWEN3VLMOE = auto() QWEN35 = auto() @@ -556,6 +564,7 @@ class MODEL_ARCH(IntEnum): ARWKV7 = auto() MAMBA = auto() MAMBA2 = auto() + MAPLE = auto() JAMBA = auto() XVERSE = auto() COMMAND_R = auto() @@ -636,6 +645,7 @@ class MODEL_ARCH(IntEnum): PADDLEOCR = auto() MIMO2 = auto() STEP35 = auto() + SPARK2_5 = auto() LLAMA_EMBED = auto() MAINCODER = auto() KIMI_LINEAR = auto() @@ -669,6 +679,7 @@ class MODEL_TENSOR(IntEnum): TOKEN_TYPES = auto() POS_EMBD = auto() OUTPUT = auto() + HRM_Z_L_INIT = auto() DENSE_2_OUT = auto() # embeddinggemma 2_Dense DENSE_3_OUT = auto() # embeddinggemma 3_Dense OUTPUT_NORM = auto() @@ -1287,6 +1298,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.QWEN3: "qwen3", MODEL_ARCH.QWEN3MOE: "qwen3moe", MODEL_ARCH.QWEN3NEXT: "qwen3next", + MODEL_ARCH.HRM_TEXT: "hrm_text", MODEL_ARCH.QWEN3VL: "qwen3vl", MODEL_ARCH.QWEN3VLMOE: "qwen3vlmoe", MODEL_ARCH.QWEN35: "qwen35", @@ -1317,6 +1329,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.ARWKV7: "arwkv7", MODEL_ARCH.MAMBA: "mamba", MODEL_ARCH.MAMBA2: "mamba2", + MODEL_ARCH.MAPLE: "maple", MODEL_ARCH.JAMBA: "jamba", MODEL_ARCH.XVERSE: "xverse", MODEL_ARCH.COMMAND_R: "command-r", @@ -1398,6 +1411,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.PADDLEOCR: "paddleocr", MODEL_ARCH.MIMO2: "mimo2", MODEL_ARCH.STEP35: "step35", + MODEL_ARCH.SPARK2_5: "spark2_5", MODEL_ARCH.LLAMA_EMBED: "llama-embed", MODEL_ARCH.MAINCODER: "maincoder", MODEL_ARCH.KIMI_LINEAR: "kimi-linear", @@ -1430,6 +1444,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.POS_EMBD: "position_embd", MODEL_TENSOR.OUTPUT_NORM: "output_norm", MODEL_TENSOR.OUTPUT: "output", + MODEL_TENSOR.HRM_Z_L_INIT: "hrm.z_l_init", MODEL_TENSOR.DENSE_2_OUT: "dense_2", # embeddinggemma 2_Dense MODEL_TENSOR.DENSE_3_OUT: "dense_3", # embeddinggemma 2_Dense MODEL_TENSOR.HC_HEAD_FN: "output_hc_fn", @@ -2326,6 +2341,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2346,6 +2362,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2369,6 +2386,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2389,6 +2407,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2434,6 +2453,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2536,6 +2556,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_TYPES, MODEL_TENSOR.ATTN_NORM_2, MODEL_TENSOR.ATTN_OUT_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -2564,6 +2585,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2593,6 +2615,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2605,6 +2628,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2632,6 +2656,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2663,6 +2688,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2678,6 +2704,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2693,6 +2720,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2707,6 +2735,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2721,6 +2750,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -2741,6 +2771,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -2757,6 +2788,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -2806,12 +2838,26 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ], + MODEL_ARCH.HRM_TEXT: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.HRM_Z_L_INIT, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], MODEL_ARCH.QWEN3VL: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -2828,6 +2874,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -2977,6 +3024,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3110,6 +3158,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3125,6 +3174,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3143,6 +3193,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ROPE_FACTORS_LONG, MODEL_TENSOR.ROPE_FACTORS_SHORT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3180,6 +3231,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3192,6 +3244,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.GEMMA2: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3208,6 +3261,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -3226,6 +3280,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -3262,6 +3317,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -3317,6 +3373,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.DENSE_2_OUT, MODEL_TENSOR.DENSE_3_OUT, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -3337,6 +3394,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3495,11 +3553,29 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.SSM_NORM, MODEL_TENSOR.SSM_OUT, ], + MODEL_ARCH.MAPLE: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + ], MODEL_ARCH.JAMBA: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3529,6 +3605,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3543,6 +3620,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3557,6 +3635,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3608,6 +3687,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.OLMO: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3620,6 +3700,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3635,6 +3716,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.SEED_OSS: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3651,6 +3733,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3667,6 +3750,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -3701,6 +3785,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3722,6 +3807,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -3784,6 +3870,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_A, MODEL_TENSOR.ATTN_Q_B, @@ -3971,6 +4058,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4047,6 +4135,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4192,6 +4281,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4206,6 +4296,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4227,6 +4318,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.SSM_D, MODEL_TENSOR.SSM_NORM, MODEL_TENSOR.SSM_OUT, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4246,6 +4338,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.SSM_D, MODEL_TENSOR.SSM_NORM, MODEL_TENSOR.SSM_OUT, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4276,6 +4369,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4291,6 +4385,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -4316,6 +4411,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -4347,6 +4443,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4361,6 +4458,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4386,6 +4484,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.SSM_D, MODEL_TENSOR.SSM_NORM, MODEL_TENSOR.SSM_OUT, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4449,6 +4548,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -4488,6 +4588,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4579,6 +4680,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -4642,6 +4744,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4657,6 +4760,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4708,6 +4812,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4722,6 +4827,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4739,6 +4845,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_NORM, # Attention components + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, # Query projection MODEL_TENSOR.ATTN_K, # Key projection MODEL_TENSOR.ATTN_V, # Value projection @@ -4771,6 +4878,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -4791,6 +4899,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -4807,6 +4916,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -4897,6 +5007,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4913,6 +5024,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4936,6 +5048,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_NORM, # operator_norm MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4956,6 +5069,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_NORM, # operator_norm MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4971,6 +5085,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -4990,6 +5105,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -5007,6 +5123,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -5024,6 +5141,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -5062,6 +5180,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -5125,6 +5244,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -5142,6 +5262,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -5157,6 +5278,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -5192,6 +5314,10 @@ class MODEL_TENSOR(IntEnum): ], MODEL_ARCH.DFLASH: [ # Dense Qwen3-style DFlash decoder + MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.FFN_POST_NORM, + MODEL_TENSOR.LAYER_OUT_SCALE, + MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, @@ -5339,6 +5465,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -5366,12 +5493,26 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ], + MODEL_ARCH.SPARK2_5: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], MODEL_ARCH.LLAMA_EMBED: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -5391,6 +5532,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, @@ -5407,6 +5549,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, @@ -5892,7 +6035,7 @@ class VisionProjectorType: GGMLQuantizationType.Q5_0: (32, 2 + 4 + 16), GGMLQuantizationType.Q5_1: (32, 2 + 2 + 4 + 16), GGMLQuantizationType.Q8_0: (32, 2 + 32), - GGMLQuantizationType.Q8_1: (32, 4 + 4 + 32), + GGMLQuantizationType.Q8_1: (32, 2 + 2 + 32), GGMLQuantizationType.Q2_K: (256, 2 + 2 + QK_K // 16 + QK_K // 4), GGMLQuantizationType.Q3_K: (256, 2 + QK_K // 4 + QK_K // 8 + 12), GGMLQuantizationType.Q4_K: (256, 2 + 2 + QK_K // 2 + 12), diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index b6ac3ba08510..6231f2d105d1 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -885,6 +885,9 @@ def add_sliding_window_pattern(self, value: int | Sequence[bool]) -> None: else: self.add_array(key, value) + def add_recurrent_layers(self, value: Sequence[bool]) -> None: + self.add_array(Keys.Attention.RECURRENT_LAYERS.format(arch=self.arch), value) + def add_rope_pattern(self, value: Sequence[bool]) -> None: self.add_array(Keys.Attention.ROPE_PATTERN.format(arch=self.arch), value) @@ -976,6 +979,18 @@ def add_residual_scale(self, value: float) -> None: def add_embedding_scale(self, value: float) -> None: self.add_float32(Keys.LLM.EMBEDDING_SCALE.format(arch=self.arch), value) + def add_hrm_layers_per_stack(self, value: int) -> None: + self.add_uint32(Keys.HRM.LAYERS_PER_STACK.format(arch=self.arch), value) + + def add_hrm_h_cycles(self, value: int) -> None: + self.add_uint32(Keys.HRM.H_CYCLES.format(arch=self.arch), value) + + def add_hrm_l_cycles(self, value: int) -> None: + self.add_uint32(Keys.HRM.L_CYCLES.format(arch=self.arch), value) + + def add_hrm_prefix_lm(self, value: bool) -> None: + self.add_bool(Keys.HRM.PREFIX_LM.format(arch=self.arch), value) + def add_adapter_count(self, count: int) -> None: self.add_uint32(Keys.Adapters.COUNT.format(arch=self.arch), count) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index d954f257c4a4..5589b8d2bd7e 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -385,6 +385,7 @@ class TensorNameMap: MODEL_TENSOR.ATTN_SINKS: ( "model.layers.{bid}.self_attn.sinks", # openai-moe "model.layers.{bid}.self_attn.attention_sink_bias", # mimov2 + "model.layers.{bid}.self_attn.learnable_sink_param", # hy-v4 ), MODEL_TENSOR.ATTN_GATE: ( @@ -392,6 +393,7 @@ class TensorNameMap: "model.layers.{bid}.linear_attn.in_proj_z", # qwen3.5 "model.layers.{bid}.self_attn.g_proj", # step3.5 head-wise attention gate "model.layers.{bid}.self_attn.output_gate", # minimax-01 + "model.layers.{bid}.self_attn.linear_gate", # hy-v4 ), # Feed-forward norm @@ -1329,6 +1331,42 @@ class TensorNameMap: "model.layers.{bid}.self_attn.index_q_norm", # MSA ), + MODEL_TENSOR.HC_ATTN_FN: ( + "model.layers.{bid}.hc_attn_layer.hc_pre.hc_fn", # hy-v4 + ), + + MODEL_TENSOR.HC_ATTN_BASE: ( + "model.layers.{bid}.hc_attn_layer.hc_pre.hc_base", # hy-v4 + ), + + MODEL_TENSOR.HC_ATTN_SCALE: ( + "model.layers.{bid}.hc_attn_layer.hc_pre.hc_scale", # hy-v4 + ), + + MODEL_TENSOR.HC_FFN_FN: ( + "model.layers.{bid}.hc_mlp_layer.hc_pre.hc_fn", # hy-v4 + ), + + MODEL_TENSOR.HC_FFN_BASE: ( + "model.layers.{bid}.hc_mlp_layer.hc_pre.hc_base", # hy-v4 + ), + + MODEL_TENSOR.HC_FFN_SCALE: ( + "model.layers.{bid}.hc_mlp_layer.hc_pre.hc_scale", # hy-v4 + ), + + MODEL_TENSOR.HC_HEAD_FN: ( + "model.hc_head.hc_head_fn", # hy-v4 + ), + + MODEL_TENSOR.HC_HEAD_BASE: ( + "model.hc_head.hc_head_base", # hy-v4 + ), + + MODEL_TENSOR.HC_HEAD_SCALE: ( + "model.hc_head.hc_head_scale", # hy-v4 + ), + ############################################################################ # TODO: these do not belong to block_mappings_cfg - move them to mappings_cfg MODEL_TENSOR.ENC_OUTPUT_NORM: ( diff --git a/gguf-py/pyproject.toml b/gguf-py/pyproject.toml index d11c34a2186d..07e6f7fee82d 100644 --- a/gguf-py/pyproject.toml +++ b/gguf-py/pyproject.toml @@ -7,7 +7,7 @@ dynamic = ["classifiers"] readme = "README.md" authors = [{name = "GGML", email = "ggml@ggml.ai"}] requires-python = '>=3.10' -dependencies = ['numpy (>=1.17)', 'tqdm (>=4.27)', 'pyyaml (>=5.1)', 'requests (>=2.25)'] +dependencies = ['numpy (>=2.2.6)', 'tqdm (>=4.27)', 'pyyaml (>=5.1)', 'requests (>=2.25)'] classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", diff --git a/grammars/README.md b/grammars/README.md index 9478b3e1b5aa..f005fc2522b8 100644 --- a/grammars/README.md +++ b/grammars/README.md @@ -146,8 +146,6 @@ You can use GBNF grammars: - For any completion endpoints, passed as the `json_schema` body field - For the `/chat/completions` endpoint, passed inside the `response_format` body field (e.g. `{"type", "json_object", "schema": {"items": {}}}` or `{ type: "json_schema", json_schema: {"schema": ...} }`) - In [llama-cli](../tools/cli) and [llama-completion](../tools/completion), passed as the `--json` / `-j` flag -- To convert to a grammar ahead of time: - - in CLI, with [examples/json_schema_to_grammar.py](../examples/json_schema_to_grammar.py) > [!NOTE] > The JSON schema is only used to constrain the model output and is not injected into the prompt. The model has no visibility into the schema, so if you want it to understand the expected structure, describe it explicitly in your prompt. This does not apply to tool calling, where schemas are injected into the prompt. @@ -187,11 +185,7 @@ llama-cli \ <summary>Show grammar</summary> -You can convert any schema in command-line with: - -```bash -examples/json_schema_to_grammar.py name-age-schema.json -``` +The schema above converts to: ``` char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) diff --git a/include/llama.h b/include/llama.h index 3726ebddd14b..344c3f08bc8d 100644 --- a/include/llama.h +++ b/include/llama.h @@ -77,6 +77,7 @@ extern "C" { LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming + LLAMA_VOCAB_TYPE_TEST = 7, // Dummy tokenizer for testing: rolling hash of fixed-size chunks -> tokens, tokens -> hex }; enum llama_rope_type { @@ -676,6 +677,8 @@ extern "C" { struct llama_model_params params); // Load a model from an open FILE pointer + // The GGUF is read from the current position, so it can be embedded in a larger file + // mmap needs the GGUF data section at a file offset to be aligned to the CPU tensor alignment (32 bytes) LLAMA_API struct llama_model * llama_model_load_from_file_ptr( FILE * file, struct llama_model_params params); @@ -839,6 +842,11 @@ extern "C" { struct llama_model * model, const char * path_lora); + // Load a LoRA adapter from an open FILE pointer, reading from its current position + LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init_from_file_ptr( + struct llama_model * model, + FILE * file); + // Functions to access the adapter's GGUF metadata scalar values // - The functions return the length of the string on success, or -1 on failure // - The output string is always null-terminated and cleared on failure @@ -1585,7 +1593,7 @@ extern "C" { LLAMA_API struct llama_sampler * llama_sampler_chain_get( struct llama_sampler * chain, int32_t i); // the total number of samplers in the chain - LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); + LLAMA_API int32_t llama_sampler_chain_n (const struct llama_sampler * chain); // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i); diff --git a/models/templates/README.md b/models/templates/README.md index 3a649b8f4dbd..022a5e278d61 100644 --- a/models/templates/README.md +++ b/models/templates/README.md @@ -23,4 +23,6 @@ These templates can be updated with the following commands: ./scripts/get_chat_template.py Qwen/Qwen3-0.6B > models/templates/Qwen-Qwen3-0.6B.jinja ./scripts/get_chat_template.py zai-org/GLM-4.5 > models/templates/zai-org-GLM-4.5.jinja ./scripts/get_chat_template.py deepseek-ai/DeepSeek-V3.1 > models/templates/deepseek-ai-DeepSeek-V3.1.jinja +./scripts/get_chat_template.py XHToken/Spark-X2.5-1.7B > models/templates/Spark2.5.jinja +./scripts/get_chat_template.py XHToken/Spark-X2.5-4B > models/templates/Spark2.5.jinja ``` diff --git a/models/templates/Spark2.5.jinja b/models/templates/Spark2.5.jinja new file mode 100644 index 000000000000..54aa34ff20d9 --- /dev/null +++ b/models/templates/Spark2.5.jinja @@ -0,0 +1,110 @@ +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} + +{%- set enable_thinking = enable_thinking | default(true) %} + +{#- Render a string or a list of text blocks. -#} +{%- macro render_content(content, context_name) %} + {%- if content is string %} + {{- content }} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- elif content is iterable and content is not mapping %} + {%- for block in content %} + {%- if block.type == 'text' %} + {{- block.text }} + {%- else %} + {{- raise_exception('Unsupported ' ~ context_name ~ ' content block type: ' ~ (block.type | string)) }} + {%- endif %} + {%- endfor %} + {%- else %} + {{- raise_exception(context_name ~ ' content must be a string or a list of text blocks') }} + {%- endif %} +{%- endmacro %} + +{#- Default system prompt. -#} +{%- set default_system = 'you are a helpful assistant.' %} + +{#- The first message-level system is placed in the initial system block. -#} +{%- set ns = namespace(initial_system='') %} +{%- if messages[0].role == 'system' %} + {%- set ns.initial_system = render_content(messages[0].content, 'system') %} +{%- endif %} + +{#- System block. -#} +{{- '<｜start▁of▁sentence｜><|System|>' + '\n' + default_system }} +{%- if tools %} + {{- '## Tools' + '\n' + 'You have access to the following functions:' + '\n' + '<tools>' }} + {%- for tool in tools %} + {{- '\n' + tool.function | tojson }} + {%- endfor %} + {{- '\n' + '</tools>' }} +{%- endif %} +{%- if ns.initial_system %} + {{- '\n\n' + ns.initial_system }} +{%- endif %} +{{- '<｜end▁of▁sentence｜>' }} + +{#- Conversation turns. -#} +{%- for message in messages %} + {%- if message.role == 'system' %} + {#- The first system message was consumed by the initial block. -#} + {%- if not loop.first %} + {{- '<｜start▁of▁sentence｜><|System|>\n' + render_content(message.content, 'system') + '<｜end▁of▁sentence｜>' }} + {%- endif %} + {%- elif message.role == 'user' %} + {{- '<｜start▁of▁sentence｜><|User|>' + render_content(message.content, 'user') + '<｜end▁of▁sentence｜>' }} + {%- elif message.role == 'assistant' %} + {%- set assistant_content = render_content(message.content, 'assistant') %} + {%- if message.reasoning_content is defined and message.reasoning_content %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- set reasoning_content = '' %} + {%- endif %} + {{- '<｜start▁of▁sentence｜><|Bot|>' }} + {%- if reasoning_content %} + {{- '<think>' + reasoning_content + '</think>' }} + {%- else %} + {{- '</think>' }} + {%- endif %} + {%- if assistant_content %} + {{- assistant_content }} + {%- endif %} + {%- if message.tool_calls is defined and message.tool_calls is not none %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function.arguments is not mapping %} + {{- raise_exception('tool_call.function.arguments must be a dictionary; normalize JSON strings before apply_chat_template') }} + {%- endif %} + {%- set args = tool_call.function.arguments %} + {{- '<tool_call>' + tool_call.function.name }} + {%- for k, v in args.items() %} + {{- '<arg_key>' ~ k ~ '</arg_key><arg_value>' ~ (v if v is string else v | tojson) ~ '</arg_value>' }} + {%- endfor %} + {{- '</tool_call>' }} + {%- endfor %} + {%- endif %} + {{- '<｜end▁of▁sentence｜>' }} + {%- elif message.role == 'tool' %} + {%- if loop.previtem is undefined or loop.previtem.role != 'tool' %} + {{- '<｜start▁of▁sentence｜><|Tool|>' }} + {%- endif %} + {{- '<tool_response>' ~ message.content ~ '</tool_response>' }} + {%- if loop.nextitem is undefined or loop.nextitem.role != 'tool' %} + {{- '<｜end▁of▁sentence｜>' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unsupported message role: ' ~ message.role) }} + {%- endif %} +{%- endfor %} + +{#- Generation prompt. -#} +{%- if add_generation_prompt %} + {{- '<｜start▁of▁sentence｜><|Bot|>' }} + {%- if enable_thinking is defined and enable_thinking %} + {{- '<think>' }} + {%- endif %} + {%- if enable_thinking is defined and not enable_thinking %} + {{- '</think>' }} + {%- endif %} +{%- endif %} diff --git a/models/templates/inclusionai-ling-3.0-flash.jinja b/models/templates/inclusionai-ling-3.0-flash.jinja new file mode 100644 index 000000000000..ed32bb978237 --- /dev/null +++ b/models/templates/inclusionai-ling-3.0-flash.jinja @@ -0,0 +1,130 @@ +{#- Bailing V3 chat template -#} +{#- Supports: thinking option, tool calling -#} + +{#- ==================== thinking option normalization ==================== -#} +{%- if enable_thinking is defined %} + {%- if enable_thinking %} + {%- set thinking_option = 'on' %} + {%- else %} + {%- set thinking_option = 'off' %} + {%- endif %} +{%- elif thinking_option is not defined %} + {%- set thinking_option = 'on' %} +{%- endif %} + +{#- ==================== preserved thinking ==================== -#} +{% set preserved_thinking = true %} + +{#- ==================== system message ==================== -#} +{{- '<role>SYSTEM</role>' }} +{%- if tools %} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n</tools>\n\nIf none of the functions can be used, point it out. If the given question lacks the parameters required by the function, also point it out.\nIf you need to use a function, for each function call, output the function name and arguments within the following XML format:\n<tool_call>{function-name}\n<arg_key>{arg-key-1}</arg_key>\n<arg_value>{arg-value-1}</arg_value>\n<arg_key>{arg-key-2}</arg_key>\n<arg_value>{arg-value-2}</arg_value>\n...\n</tool_call>\n" }} + {%- if messages[0].role == 'system' and messages[0].content is string and ('detailed thinking on' in messages[0].content or 'detailed thinking off' in messages[0].content) %} + {{- '<|role_end|>' }} + {%- else %} + {{- 'detailed thinking ' + thinking_option + '<|role_end|>' }} + {%- endif %} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if 'detailed thinking on' in messages[0].content or 'detailed thinking off' in messages[0].content %} + {{- messages[0].content + '<|role_end|>' }} + {%- else %} + {{- messages[0].content + '\n' }} + {{- 'detailed thinking ' + thinking_option + '<|role_end|>' }} + {%- endif %} + {% else %} + {{- 'detailed thinking ' + thinking_option + '<|role_end|>' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if message.role == "user" %} + {{- '<role>HUMAN</role>' + message.content + '<|role_end|>' }} + {%- elif message.role == "system" and not loop.first %} + {{- '<role>SYSTEM</role>' + message.content + '<|role_end|>' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string and message.reasoning_content != '' %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '</think>' in content %} + {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %} + {%- set content = content.split('</think>')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if preserved_thinking or loop.index0 > ns.last_query_index %} + {%- if reasoning_content != '' %} + {{- '<role>ASSISTANT</role>' + '\n<think>' + reasoning_content.strip('\n') + '</think>' + content.lstrip('\n') }} + {%- else %} + {{- '<role>ASSISTANT</role>\n<think></think>' + content }} + {%- endif %} + {%- else %} + {{- '<role>ASSISTANT</role>\n<think></think>' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- set tc = tool_call %} + {%- if tool_call.function %} + {%- set tc = tool_call.function %} + {%- endif %} + {{- '<tool_call>' + tc.name }} + {% set _args = tc.arguments %} + {%- for k, v in _args.items() %} + {{- '<arg_key>' + k + '</arg_key>' }} + {{- '\n<arg_value>' }} + {%- if v is string %} + {{- v }} + {%- else %} + {{- v | tojson(ensure_ascii=False) }} + {%- endif %} + {{- '</arg_value>' }} + {%- endfor %} + {{- '\n</tool_call>' }} + {%- endfor %} + {%- endif %} + {{- '<|role_end|>' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<role>OBSERVATION</role>' }} + {%- endif %} + {{- '\n<tool_response>\n' }} + {{- content }} + {{- '\n</tool_response>' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|role_end|>' }} + {%- endif %} + {%- endif %} +{%- endfor %} + +{#- ==================== generation prompt ==================== -#} +{%- if add_generation_prompt %} + {{- '<role>ASSISTANT</role>' }} + {%- if thinking_option == 'on' %} + {{- '\n<think>' }} + {%- elif thinking_option == 'off' %} + {{- '\n<think></think>' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/pocs/CMakeLists.txt b/pocs/CMakeLists.txt index d49d14dee435..93c76d609a61 100644 --- a/pocs/CMakeLists.txt +++ b/pocs/CMakeLists.txt @@ -8,7 +8,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) if (EMSCRIPTEN) else() - if (NOT GGML_BACKEND_DL) + if (NOT GGML_BACKEND_DL AND GGML_CPU) add_subdirectory(vdot) endif() endif() diff --git a/pyproject.toml b/pyproject.toml index bbcb9bddcdf8..028bc8cf4f4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ readme = "README.md" authors = [{name = "GGML", email = "ggml@ggml.ai"}] requires-python = '>=3.10,<3.15' dependencies = [ - 'numpy (>=1.26.4,<3.0.0)', + 'numpy (>=2.2.6,<3.0.0)', 'sentencepiece (>=0.1.98,<0.3.0)', 'transformers (==4.57.6)', 'protobuf (>=4.21.0,<5.0.0)', diff --git a/requirements/requirements-convert_legacy_llama.txt b/requirements/requirements-convert_legacy_llama.txt index 28221fad0ce9..edc945c1cb67 100644 --- a/requirements/requirements-convert_legacy_llama.txt +++ b/requirements/requirements-convert_legacy_llama.txt @@ -1,4 +1,4 @@ -numpy~=1.26.4 +numpy~=2.2.6 sentencepiece>=0.1.98,<0.3.0 transformers==4.57.6 diff --git a/requirements/requirements-gguf_editor_gui.txt b/requirements/requirements-gguf_editor_gui.txt index fd253364e152..93fe087223eb 100644 --- a/requirements/requirements-gguf_editor_gui.txt +++ b/requirements/requirements-gguf_editor_gui.txt @@ -1,3 +1,3 @@ -numpy~=1.26.4 +numpy~=2.2.6 PySide6~=6.9.0 gguf>=0.17.0 diff --git a/requirements/requirements-server-bench.txt b/requirements/requirements-server-bench.txt index fb3b0d2664b0..0b065b1ea4c1 100644 --- a/requirements/requirements-server-bench.txt +++ b/requirements/requirements-server-bench.txt @@ -1,5 +1,5 @@ datasets~=4.8.0 matplotlib~=3.10.0 -numpy~=1.26.4 +numpy~=2.2.6 requests~=2.32.3 tqdm~=4.67.1 diff --git a/requirements/requirements-tool_bench.txt b/requirements/requirements-tool_bench.txt index 3e6f824165c4..ba865115026d 100644 --- a/requirements/requirements-tool_bench.txt +++ b/requirements/requirements-tool_bench.txt @@ -1,7 +1,7 @@ aiohttp~=3.9.3 pytest~=8.3.3 matplotlib~=3.10.0 -numpy~=1.26.4 +numpy~=2.2.6 openai~=2.14.0 pandas~=2.2.3 prometheus-client~=0.20.0 diff --git a/scripts/build-profile.ps1 b/scripts/build-profile.ps1 new file mode 100644 index 000000000000..410ead39d5cc --- /dev/null +++ b/scripts/build-profile.ps1 @@ -0,0 +1,136 @@ +# Compile-time profiling using clang -ftime-trace + ClangBuildAnalyzer. +# +# Usage: +# .\scripts\build-profile.ps1 [-Full] [-Jobs N] +# +# -Full : include Server, Tools, and Tests (default: minimal build) +# -Jobs : number of parallel jobs (default: all cores) +# +# Requires ClangBuildAnalyzer: +# https://github.com/aras-p/ClangBuildAnalyzer + +param( + [switch]$Full, + [int]$Jobs = [Environment]::ProcessorCount +) + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$RootDir = Split-Path -Parent $ScriptDir + +if ($Full) { + $BuildDir = Join-Path $RootDir "build-profile-full" + $Report = Join-Path $BuildDir "profile-report-full.txt" +} else { + $BuildDir = Join-Path $RootDir "build-profile-baseline" + $Report = Join-Path $BuildDir "profile-report.txt" +} + +$OutputBin = Join-Path $BuildDir "clang_analysis.bin" + +if (-not (Get-Command clang++ -ErrorAction SilentlyContinue)) { + Write-Error "clang++ not found" + exit 1 +} + +if (-not (Get-Command ninja -ErrorAction SilentlyContinue)) { + Write-Error "ninja not found (required so cmake does not fall back to the Visual Studio/MSVC generator)" + exit 1 +} + +if (-not (Get-Command ClangBuildAnalyzer -ErrorAction SilentlyContinue)) { + Write-Error "ClangBuildAnalyzer not found`n https://github.com/aras-p/ClangBuildAnalyzer/releases" + exit 1 +} + +$ClangVer = (clang++ --version | Select-Object -First 1) +Write-Host "compiler : $ClangVer" +Write-Host "build dir: $BuildDir" +Write-Host "output : $OutputBin" +Write-Host "jobs : $Jobs" +Write-Host "" + +if (Get-Command ccache -ErrorAction SilentlyContinue) { + Write-Host "clearing ccache..." + ccache -C -z +} + +$env:CCACHE_DISABLE = "1" + +$TestsFlag = if ($Full) { "ON" } else { "OFF" } +$ToolsFlag = if ($Full) { "ON" } else { "OFF" } +$ServerFlag = if ($Full) { "ON" } else { "OFF" } + +cmake --fresh ` + -S $RootDir ` + -B $BuildDir ` + -G "Ninja" ` + -DCMAKE_BUILD_TYPE=Release ` + -DCMAKE_C_COMPILER=clang ` + -DCMAKE_CXX_COMPILER=clang++ ` + -DCMAKE_C_FLAGS="-ftime-trace" ` + -DCMAKE_CXX_FLAGS="-ftime-trace" ` + -DGGML_CCACHE=OFF ` + -DGGML_OPENMP=ON ` + -DGGML_NATIVE=OFF ` + "-DLLAMA_BUILD_TESTS=$TestsFlag" ` + -DLLAMA_BUILD_EXAMPLES=OFF ` + "-DLLAMA_BUILD_TOOLS=$ToolsFlag" ` + "-DLLAMA_BUILD_SERVER=$ServerFlag" ` + -DLLAMA_BUILD_APP=OFF + +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +$StrayTrace = Join-Path $RootDir "-.json" +if (Test-Path $StrayTrace) { + Remove-Item $StrayTrace -Force +} + +Write-Host "" +Write-Host "Initializing ClangBuildAnalyzer..." +ClangBuildAnalyzer --start $BuildDir +Write-Host "" + +Write-Host "building..." +Write-Host "" + +$StartTime = Get-Date + +cmake --build $BuildDir --clean-first -j $Jobs + +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +$Elapsed = (Get-Date) - $StartTime + +Write-Host "" +Write-Host ("build time: {0}s ({1}m {2}s)" -f [int]$Elapsed.TotalSeconds, [int]$Elapsed.TotalMinutes, $Elapsed.Seconds) +Write-Host "" + +Write-Host "Aggregating profile metrics..." +ClangBuildAnalyzer --stop $BuildDir $OutputBin | Out-Null + +Write-Host "" +Write-Host ("=" * 80) + +$TUs = "?" +if (Test-Path $Report) { + $Match = Select-String -Path $Report -Pattern "Compilation \((\d+)" | Select-Object -First 1 + if ($Match) { $TUs = $Match.Matches[0].Groups[1].Value } +} + +ClangBuildAnalyzer --analyze $OutputBin | Tee-Object -FilePath $Report + +Write-Host "" +Write-Host "translation units: $TUs" +Write-Host "" +Write-Host "largest trace files (top 20 by size):" + +Get-ChildItem -Path $BuildDir -Recurse -Filter "*.json" | + Where-Object { $_.Name -ne "compile_commands.json" } | + Sort-Object Length -Descending | + Select-Object -First 20 | + ForEach-Object { "{0,8:F1} KB {1}" -f ($_.Length / 1024), $_.FullName } + +Write-Host "" +Write-Host "ClangBuildAnalyzer report was generated: $Report" diff --git a/scripts/build-profile.sh b/scripts/build-profile.sh new file mode 100755 index 000000000000..94299498909b --- /dev/null +++ b/scripts/build-profile.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Compile-time profiling using clang -ftime-trace + ClangBuildAnalyzer. +# +# Usage: +# ./scripts/build-profile.sh [--full] [-jN] +# +# --full: include Server, Tools, and Tests (default: minimal build) +# -jN : number of parallel jobs (default: all cores) +# +# Requires ClangBuildAnalyzer: +# macOS: brew install clang-build-analyzer +# Linux: https://github.com/aras-p/ClangBuildAnalyzer.git + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" + +FULL=0 +JOBS="-j$(nproc 2>/dev/null || sysctl -n hw.ncpu)" + +for arg in "$@"; do + case "${arg}" in + --full) FULL=1 ;; + -j*) JOBS="${arg}" ;; + *) echo "error: unknown argument: ${arg}" >&2; exit 1 ;; + esac +done + +if [ "${FULL}" -eq 1 ]; then + BUILD_DIR="${ROOT_DIR}/build-profile-full" + REPORT="${BUILD_DIR}/profile-report-full.txt" +else + BUILD_DIR="${ROOT_DIR}/build-profile-baseline" + REPORT="${BUILD_DIR}/profile-report.txt" +fi + +OUTPUT_BIN="${BUILD_DIR}/clang_analysis.bin" + +if ! command -v clang++ &>/dev/null; then + echo "error: clang++ not found" >&2 + exit 1 +fi + +if ! command -v ClangBuildAnalyzer &>/dev/null; then + echo "error: ClangBuildAnalyzer not found" >&2 + echo " brew install clangbuildanalyzer (macOS)" >&2 + echo " or: https://github.com/aras-p/ClangBuildAnalyzer/releases" >&2 + exit 1 +fi + +CLANG_VER=$(clang++ --version | head -1) +echo "compiler : ${CLANG_VER}" +echo "build dir: ${BUILD_DIR}" +echo "output : ${OUTPUT_BIN}" +echo "jobs : ${JOBS}" +echo + +if command -v ccache &>/dev/null; then + echo "clearing ccache..." + ccache -C -z +fi + +export CCACHE_DISABLE=1 + +cmake --fresh \ + -S "${ROOT_DIR}" \ + -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_FLAGS="-ftime-trace" \ + -DCMAKE_CXX_FLAGS="-ftime-trace" \ + -DGGML_CCACHE=OFF \ + -DGGML_OPENMP=ON \ + -DGGML_NATIVE=OFF \ + -DLLAMA_BUILD_TESTS=$([ "${FULL}" -eq 1 ] && echo ON || echo OFF) \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=$([ "${FULL}" -eq 1 ] && echo ON || echo OFF) \ + -DLLAMA_BUILD_SERVER=$([ "${FULL}" -eq 1 ] && echo ON || echo OFF) \ + -DLLAMA_BUILD_APP=OFF + +echo + +echo "Initializing ClangBuildAnalyzer..." +ClangBuildAnalyzer --start "${BUILD_DIR}" +echo + +echo "building..." +echo + +START=$(date +%s) + +cmake --build "${BUILD_DIR}" --clean-first "${JOBS}" + +END=$(date +%s) +ELAPSED=$((END - START)) + +echo +printf "build time: %ds (%dm %ds)\n" "${ELAPSED}" "$((ELAPSED / 60))" "$((ELAPSED % 60))" +echo + +echo "Aggregating profile metrics..." +ClangBuildAnalyzer --stop "${BUILD_DIR}" "${OUTPUT_BIN}" > /dev/null + +echo +echo "================================================================================" +TUS=$(grep -oP "Compilation \(\K[0-9]+" "${REPORT}" 2>/dev/null || echo "?") +ClangBuildAnalyzer --analyze "${OUTPUT_BIN}" | tee "${REPORT}" + +echo +echo "translation units: ${TUS}" +echo +echo "largest trace files (top 20 by size):" +find "${BUILD_DIR}" -name "*.json" ! -name "compile_commands.json" \ + | xargs ls -l 2>/dev/null \ + | awk 'NF>5 {print $5, $NF}' \ + | sort -rn \ + | awk 'NR<=20 {printf "%8.1f KB %s\n", $1/1024, $2}' + +echo +echo "ClangBuildAnalyzer report was generated: ${REPORT}" diff --git a/scripts/check-apiabi-compat.sh b/scripts/check-apiabi-compat.sh new file mode 100755 index 000000000000..8075aaf9f1fe --- /dev/null +++ b/scripts/check-apiabi-compat.sh @@ -0,0 +1,290 @@ +#!/bin/sh +# Check for backwards-incompatible API and ABI changes between two builds +# +# Backwards-incompatible API changes, such as removing a value from an enum, +# are checked by abi-compliance-checker. Such changes can break compilation of +# existing programs. +# +# Backwards-incompatible ABI changes, such as the removal of a public function, +# are checked by libigail-tools. Such changes could break run-time dynamic +# linking of existing binaries. (We don't use a-c-c for ABI checks because it +# needs a debug build, whereas abigail does not.) +# +# Commands: +# --generate <build-dir>: Creates API/ABI dumps in <build-dir> +# <build-dir> is expected to be a CMake build result +# --check <dir1> <dir2>: Compares dumps in <dir1> and <dir2> +# Comparison exit codes +# 0: all good +# 1: backwards-incompatible changes found +# +# Options: +# --include-path <dir>: a-c-c calls gcc on headers; use this option to add +# directories to gcc's search path +# +# +# This script would typically be used before cutting a release: +# +# 1. Generate API/ABI dump for the old version +# +# $ check-apiabi-compat.sh --generate <build-dir-old> libfoo [ libbar ...] +# +# 2. <update source> +# +# 3. Generate API/ABI dump for the new version +# +# $ check-apiabi-compat.sh --generate <build-dir-new> libfoo [ libbar ...] +# +# 4. Compare the two dumps +# +# $ check-apiabi-compat.sh --check <old-build-dir> <new-build-dir> +# +# If the check exits 0, all is fine. Otherwise, backwards-incompatible +# changes were found, and the librar(ies) need a SOVER bump. +set -eu + +# Preconditions +if ! command -v abi-compliance-checker >/dev/null 2>&1; then + echo "abi-compliance-checker is not installed." >&2 + exit 1 +elif ! command -v abidw >/dev/null 2>&1; then + echo "abigail-tools are not installed." >&2 + exit 1 +fi + +# Some generic functions +usage() { + echo "Usage: $0 [ --include-path <dir> ] --generate <build-dir> libXXX [ libYYY ... ]" >&2 + echo " $0 [ --strict ] --check <old-build-dir> <new-build-dir>" >&2 + echo "" >&2 + echo " --strict: fail on any API/ABI change, including backwards-compatible additions" >&2 +} + +get_cmake_project_name() { + sed -nr 's/^project\("(.*)".*$/\1/p' CMakeLists.txt +} + +get_cmake_version() { + major="$(sed -nr 's/^set\([A-Z]+_VERSION_MAJOR ([0-9]+)\)$/\1/p' CMakeLists.txt)" + minor="$(sed -nr 's/^set\([A-Z]+_VERSION_MINOR ([0-9]+)\)$/\1/p' CMakeLists.txt)" + patch="$(sed -nr 's/^set\([A-Z]+_VERSION_PATCH ([0-9]+)\)$/\1/p' CMakeLists.txt)" + echo "$major.$minor.$patch" +} + +# Option parsing and validation +DO_GEN=0 +DO_CHECK=0 +STRICT=0 +BUILD_DIR= +BUILD_DIR_NEW= +INCLUDE_PATHS= +while [ "$#" -gt 0 ]; do + case "$1" in + --generate=*) + DO_GEN=1 + BUILD_DIR="${1#*=}" + shift + ;; + --generate) + DO_GEN=1 + if [ -z "${2:-}" ]; then + usage + exit 1 + fi + BUILD_DIR="$2" + shift 2 + ;; + --check) + DO_CHECK=1 + if [ -z "${2:-}" ] || [ -z "${3:-}" ]; then + usage + exit 1 + elif ! [ -d "$2" ]; then + echo "$2 is not a directory." >&2 + exit 1 + elif ! [ -d "$3" ]; then + echo "$3 is not a directory." >&2 + exit 1 + fi + BUILD_DIR="$2" + BUILD_DIR_NEW="$3" + shift 3 + ;; + --include-path=*) + INCLUDE_PATHS="$INCLUDE_PATHS ${1#*=}" + shift + ;; + --include-path) + if [ -z "${2:-}" ]; then + usage + exit 1 + fi + INCLUDE_PATHS="$INCLUDE_PATHS $2" + shift 2 + ;; + --strict) + STRICT=1 + shift + ;; + -h | --help) + usage + exit 1 + ;; + -?*) + usage + exit 1 + ;; + *) + break + ;; + esac +done +if [ $((DO_GEN + DO_CHECK)) -gt 1 ]; then + echo "Can only use one --command." >&2 + exit +fi +PROJECT_NAME="$(get_cmake_project_name)" +PROJECT_VERSION="$(get_cmake_version)" +LIB_NAMES="" +while [ "$#" -gt 0 ]; do + if [ "${1#lib}" = "$1" ]; then + echo "Library to check must start with libXXX." >&2 + exit 1 + fi + LIB_NAMES="$LIB_NAMES $1" + shift +done + +dump_current_api() { + echo "Dumping API..." + + DESCRIPTOR="$BUILD_DIR/apiabi/acc-descriptor.xml" + mkdir -p "$BUILD_DIR/apiabi" + cat >"$DESCRIPTOR" <<EOF +<version>$PROJECT_VERSION</version> +<headers>include</headers> +<add_include_paths>$INCLUDE_PATHS</add_include_paths> +EOF + + # This addresses a bug between a-c-c and universal-ctags, manifested when + # a name is use both for a tag and a function name + mkdir -p "$BUILD_DIR/apiabi/.ctags.d" + echo "--fields=-t" >"$BUILD_DIR/apiabi/.ctags.d/acc.ctags" + + # Change HOME so that .ctags.d gets picked up by universal-ctags, if used + HOME="$BUILD_DIR/apiabi" abi-compliance-checker \ + -headers-only \ + -lib "$PROJECT_NAME" \ + -dump "$DESCRIPTOR" \ + -log-path "$BUILD_DIR/apiabi/acc.log" \ + -dump-path "$BUILD_DIR/apiabi/api.dump" + # acc generates this file with an ancient timestamp, which confuses gzip + touch "$BUILD_DIR/apiabi/api.dump" +} + +dump_current_abi() { + echo "Dumping ABIs ..." + mkdir -p "$BUILD_DIR/apiabi" + # The suppressions are needed to avoid including all the internal C++ + # symbols, and system types + cat >"$BUILD_DIR/apiabi/abidw.suppress" <<EOF +[suppress_function] + +label = suppress internal C++ mangled functions +symbol_name_regexp = ^_Z +drop = yes + +[suppress_variable] +label = suppress internal C++ mangled variables +symbol_name_regexp = ^_Z +drop = yes + +[suppress_type] +label = Suppress types outside of our own source +source_location_not_regexp = ^include/ +drop = yes +EOF + + # In abidw 2.5, handling of undefined stuff was changed a bit + abidw_version="$(abidw --version | sed -r 's/^abidw: ([0-9]+\.[0-9]+).*$/\1/')" + abidw_major="${abidw_version%.*}" + abidw_minor="${abidw_version#*.}" + if [ "$abidw_major" -gt 2 ] || [ "$abidw_minor" -gt 4 ]; then + abidw_undefined_syms_options="--no-load-undefined-interfaces" + else + abidw_undefined_syms_options="--drop-undefined-syms" + fi + + for lib_name in $LIB_NAMES; do + # Depending on where add_library resides, the libraries can end up in + # build/src or build/bin + lib_path="$BUILD_DIR/src/$lib_name.so" + if ! [ -f "$lib_path" ]; then + lib_path="$BUILD_DIR/bin/$lib_name.so" + if ! [ -f "$lib_path" ]; then + echo "Cannot find library $lib_name.so" >&2 + exit 1 + fi + fi + abidw \ + --headers-dir include \ + "$abidw_undefined_syms_options" \ + --suppressions "$BUILD_DIR/apiabi/abidw.suppress" \ + --out-file "${BUILD_DIR}/apiabi/$lib_name.abi.xml" \ + "$lib_path" + done +} + +# Run the actual commands +if [ "$DO_GEN" -eq 1 ]; then + dump_current_api + dump_current_abi + exit 0 +elif [ "$DO_CHECK" -eq 1 ]; then + # From here on, we don't want to exit on first error + set +e + + abi-compliance-checker \ + -strict \ + -source \ + -library "$PROJECT_NAME" \ + -old "$BUILD_DIR/apiabi/api.dump" \ + -new "$BUILD_DIR_NEW/apiabi/api.dump" \ + -src-report-path "$BUILD_DIR_NEW/apiabi/api_compat_report.html" + API_RESULT=$? + + ABI_RESULT=0 + for xml_file in "$BUILD_DIR/apiabi/"lib*.abi.xml; do + xml_file_new="$BUILD_DIR_NEW/apiabi/$(basename "$xml_file")" + + if ! [ -f "$xml_file_new" ]; then + echo "Cannot compare, missing file: $xml_file_new" >&2 + exit 1 + fi + + abidiff "$xml_file" "$xml_file_new" + res=$? + [ "$((res & 8))" -ne 0 ] && ABI_RESULT=1 + # check bit 2 for compatible ABI changes (like new symbols) and if + # STRICT is set then handle this as an error. + [ "$STRICT" -eq 1 ] && [ "$((res & 4))" -ne 0 ] && ABI_RESULT=1 + done + + if [ "$API_RESULT" -gt 0 ]; then + if [ "$STRICT" -eq 1 ]; then + echo "ERROR: API changed — a minor version bump is required." >&2 + else + echo "ERROR: API changed with backwards-incompatible problems — a major version bump is required." >&2 + fi + fi + if [ "$ABI_RESULT" -gt 0 ]; then + if [ "$STRICT" -eq 1 ]; then + echo "ERROR: ABI changed — a minor version bump is required." >&2 + else + echo "ERROR: ABI changed with backwards-incompatible problems — a major version bump is required." >&2 + fi + fi + if [ "$((API_RESULT + ABI_RESULT))" -gt 0 ]; then + exit 1 + fi +fi diff --git a/scripts/check-release-apiabi.sh b/scripts/check-release-apiabi.sh new file mode 100755 index 000000000000..d31d24041898 --- /dev/null +++ b/scripts/check-release-apiabi.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# Check API/ABI compatibility between the previous release tag and current HEAD. +# +# Finds the most recent vX.Y.Z tag, checks it out in a temporary git worktree, +# builds both versions with shared libs enabled, and uses check-apiabi-compat.sh +# to compare the results. +# +# Exit codes: +# 0: compatible, or check was skipped +# 1: backwards-incompatible changes found, or build failed +# +# Options: +# --tag <version>: compare against this tag instead of the latest release +# +# Environment: +# SKIP_APIABI_CHECK: set to 1 or true to skip +# APIABI_COMPARE_TAG: equivalent to --tag (used by CI) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +usage() { + echo "Usage: $0 [--tag <version>]" >&2 + echo " --tag <version> Compare against this release tag (default: latest)" >&2 +} + +COMPARE_TAG="${APIABI_COMPARE_TAG:-}" +while [[ "$#" -gt 0 ]]; do + case "$1" in + --tag) + if [[ -z "${2:-}" ]]; then usage; exit 1; fi + COMPARE_TAG="$2" + shift 2 + ;; + --tag=*) + COMPARE_TAG="${1#*=}" + shift + ;; + -h | --help) + usage; exit 0 + ;; + *) + usage; exit 1 + ;; + esac +done + +if [[ "${SKIP_APIABI_CHECK:-}" == "1" || "${SKIP_APIABI_CHECK:-}" == "true" ]]; then + echo "SKIP_APIABI_CHECK is set - skipping API/ABI compatibility check" + exit 0 +fi + +if ! command -v abi-compliance-checker >/dev/null 2>&1 || ! command -v abidw >/dev/null 2>&1; then + echo "Warning: abi-compliance-checker or abigail-tools not installed - skipping API/ABI check" + exit 0 +fi + +discover_libs() { + local build_dir="$1" + local libs=() + for dir in "$build_dir/src" "$build_dir/bin"; do + [[ -d "$dir" ]] || continue + for f in "$dir"/lib*.so; do + [[ -f "$f" ]] && libs+=("$(basename "$f" .so)") + done + done + echo "${libs[@]}" +} + +if [[ -n "${COMPARE_TAG}" ]]; then + PREV_TAG="${COMPARE_TAG}" + if ! git -C "$REPO_ROOT" rev-parse --verify "${PREV_TAG}^{}" >/dev/null 2>&1; then + echo "Error: tag '${PREV_TAG}' not found in repository." >&2 + exit 1 + fi +else + PREV_TAG=$(git -C "$REPO_ROOT" tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -n 1 || true) + if [[ -z "${PREV_TAG}" ]]; then + echo "Warning: no previous release tag found - skipping API/ABI check" + exit 0 + fi +fi +OLD_VERSION="${PREV_TAG#v}" +OLD_MAJOR="${OLD_VERSION%%.*}" +OLD_MINOR="${OLD_VERSION#*.}"; OLD_MINOR="${OLD_MINOR%%.*}" + +NEW_MAJOR=$(grep "set(LLAMA_VERSION_MAJOR" "$REPO_ROOT/CMakeLists.txt" | sed 's/.*MAJOR \([0-9]*\).*/\1/') +NEW_MINOR=$(grep "set(LLAMA_VERSION_MINOR" "$REPO_ROOT/CMakeLists.txt" | sed 's/.*MINOR \([0-9]*\).*/\1/') + +if [[ "$NEW_MAJOR" -gt "$OLD_MAJOR" ]]; then + echo "Major version increment ($OLD_MAJOR -> $NEW_MAJOR): API/ABI breaking changes are expected, skipping compatibility check." + exit 0 +fi + +CHECK_FLAGS=() +if [[ "$NEW_MINOR" -eq "$OLD_MINOR" ]]; then + echo "Patch version bump detected: checking for any API/ABI changes (a minor bump is required if any are found)..." + CHECK_FLAGS+=(--strict) +else + echo "Minor version bump detected: checking for backwards-incompatible API/ABI changes..." +fi + +echo "Checking API/ABI compatibility against ${PREV_TAG}..." + +WORKTREE_DIR=$(mktemp -d) +BUILD_OLD=$(mktemp -d) +BUILD_NEW=$(mktemp -d) + +cleanup() { + git -C "$REPO_ROOT" worktree remove --force "$WORKTREE_DIR" 2>/dev/null || true + rm -rf "$WORKTREE_DIR" "$BUILD_OLD" "$BUILD_NEW" +} +trap cleanup EXIT + +git -C "$REPO_ROOT" worktree add "$WORKTREE_DIR" "$PREV_TAG" + +cmake -S "$WORKTREE_DIR" -B "$BUILD_OLD" -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +cmake --build "$BUILD_OLD" --parallel "$(nproc)" +OLD_LIBS=($(discover_libs "$BUILD_OLD")) +echo "Libraries found in old build: ${OLD_LIBS[*]}" + +cmake -S "$REPO_ROOT" -B "$BUILD_NEW" -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo +cmake --build "$BUILD_NEW" --parallel "$(nproc)" +NEW_LIBS=($(discover_libs "$BUILD_NEW")) +echo "Libraries found in new build: ${NEW_LIBS[*]}" + +(cd "$WORKTREE_DIR" && "$SCRIPT_DIR/check-apiabi-compat.sh" --include-path ggml/include --generate "$BUILD_OLD" "${OLD_LIBS[@]}") +(cd "$REPO_ROOT" && "$SCRIPT_DIR/check-apiabi-compat.sh" --include-path ggml/include --generate "$BUILD_NEW" "${NEW_LIBS[@]}") +(cd "$REPO_ROOT" && "$SCRIPT_DIR/check-apiabi-compat.sh" "${CHECK_FLAGS[@]}" --check "$BUILD_OLD" "$BUILD_NEW") diff --git a/scripts/hip/gcn-cdna-vgpr-check.py b/scripts/hip/gcn-cdna-vgpr-check.py index 40fb789417c3..f660735f401f 100644 --- a/scripts/hip/gcn-cdna-vgpr-check.py +++ b/scripts/hip/gcn-cdna-vgpr-check.py @@ -64,6 +64,7 @@ def main(): '_ZL12rwkv_wkv_f32ILi128EEviiiiPKfS1_S1_S1_S1_S1_Pf', '_ZL9mul_mat_qIL9ggml_type10ELi64ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_', '_ZL9mul_mat_qIL9ggml_type42ELi128ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil' } functions = parse_log_file(log_file) diff --git a/scripts/jinja/jinja-tester.py b/scripts/jinja/jinja-tester.py index a83f025411ae..6d36ecfa575d 100755 --- a/scripts/jinja/jinja-tester.py +++ b/scripts/jinja/jinja-tester.py @@ -20,7 +20,6 @@ from jinja2 import TemplateSyntaxError from jinja2.sandbox import ImmutableSandboxedEnvironment from datetime import datetime -from typing import Callable def format_template_content(template_content): @@ -396,7 +395,7 @@ def raise_exception(text: str) -> str: ensure_ascii=ensure_ascii, ) ) - env.globals["strftime_now"]: Callable[[str], str] = lambda format: datetime.now().strftime(format) + env.globals["strftime_now"] = lambda format: datetime.now().strftime(format) # ty: ignore[invalid-assignment, invalid-argument-type] env.globals["raise_exception"] = raise_exception # ty: ignore[invalid-assignment] try: template = env.from_string(template_str) diff --git a/scripts/make-release-checks.sh b/scripts/make-release-checks.sh index bc575e5a46f4..2b60e870faab 100755 --- a/scripts/make-release-checks.sh +++ b/scripts/make-release-checks.sh @@ -22,9 +22,9 @@ for arg in "$@"; do esac done -MAJOR=$(grep "set(LLAMA_VERSION_MAJOR" "$REPO_ROOT/CMakeLists.txt" | grep -oP '\d+') -MINOR=$(grep "set(LLAMA_VERSION_MINOR" "$REPO_ROOT/CMakeLists.txt" | grep -oP '\d+') -PATCH=$(grep "set(LLAMA_VERSION_PATCH" "$REPO_ROOT/CMakeLists.txt" | grep -oP '\d+') +MAJOR=$(grep "set(LLAMA_VERSION_MAJOR" "$REPO_ROOT/CMakeLists.txt" | sed 's/.*MAJOR \([0-9]*\).*/\1/') +MINOR=$(grep "set(LLAMA_VERSION_MINOR" "$REPO_ROOT/CMakeLists.txt" | sed 's/.*MINOR \([0-9]*\).*/\1/') +PATCH=$(grep "set(LLAMA_VERSION_PATCH" "$REPO_ROOT/CMakeLists.txt" | sed 's/.*PATCH \([0-9]*\).*/\1/') VERSION="v${MAJOR}.${MINOR}.${PATCH}" echo "Determined version: ${VERSION}" if [[ -n "${GITHUB_OUTPUT:-}" ]]; then @@ -91,9 +91,9 @@ else fi fi -MAJOR=$(grep "set(GGML_VERSION_MAJOR" "$REPO_ROOT/ggml/CMakeLists.txt" | grep -oP '\d+') -MINOR=$(grep "set(GGML_VERSION_MINOR" "$REPO_ROOT/ggml/CMakeLists.txt" | grep -oP '\d+') -PATCH=$(grep "set(GGML_VERSION_PATCH" "$REPO_ROOT/ggml/CMakeLists.txt" | grep -oP '\d+') +MAJOR=$(grep "set(GGML_VERSION_MAJOR" "$REPO_ROOT/ggml/CMakeLists.txt" | sed 's/.*MAJOR \([0-9]*\).*/\1/') +MINOR=$(grep "set(GGML_VERSION_MINOR" "$REPO_ROOT/ggml/CMakeLists.txt" | sed 's/.*MINOR \([0-9]*\).*/\1/') +PATCH=$(grep "set(GGML_VERSION_PATCH" "$REPO_ROOT/ggml/CMakeLists.txt" | sed 's/.*PATCH \([0-9]*\).*/\1/') GGML_VERSION="v${MAJOR}.${MINOR}.${PATCH}" echo "Local ggml version: ${GGML_VERSION}" @@ -120,6 +120,68 @@ else fi fi +echo "Checking container images for commit ${SHA}..." +NIGHTLY_TAG="$(git tag --points-at "${SHA}" | grep -E '(^|-)b[0-9]+(-[0-9a-f]{7})?$' | head -n 1 || true)" +if [[ -z "${NIGHTLY_TAG}" ]]; then + echo "Warning: no nightly tag points at ${SHA} - skipping container image check" +elif [[ -z "${GITHUB_REPOSITORY:-}" ]]; then + echo "Warning: GITHUB_REPOSITORY not set - skipping container image check (local run)" +else + CONTAINER_REPO="${GITHUB_REPOSITORY,,}" # lower-case owner/repo for ghcr.io + GHCR_TOKEN="$(curl -fsSL \ + "https://ghcr.io/token?scope=repository:${CONTAINER_REPO}:pull&service=ghcr.io" \ + | grep -oP '"token"\s*:\s*"\K[^"]+')" + + VARIANTS=("" "-cuda" "-cuda13" "-vulkan" "-rocm" "-intel" "-musa" "-openvino") + TYPES=("full" "light" "server") + CONTAINER_ERR="" + for type in "${TYPES[@]}"; do + for variant in "${VARIANTS[@]}"; do + tag="${type}${variant}-${NIGHTLY_TAG}" + STATUS="$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer ${GHCR_TOKEN}" \ + -H "Accept: application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json" \ + "https://ghcr.io/v2/${CONTAINER_REPO}/manifests/${tag}")" + if [[ "${STATUS}" == "200" ]]; then + echo " ${tag} - OK" + else + echo " ${tag} - MISSING" + CONTAINER_ERR+=" ${tag}" + fi + done + done + + if [[ -n "${CONTAINER_ERR}" ]]; then + if [[ "$DRY_RUN" == "true" ]]; then + echo "Warning: missing container images for ${NIGHTLY_TAG}:${CONTAINER_ERR} (dry run, continuing)." + CHECKS_PASSED=false + else + echo "Error: missing container images for ${NIGHTLY_TAG}:${CONTAINER_ERR}" + echo "The Docker workflow must complete successfully before making a release." + exit 1 + fi + else + echo "All container images found for ${NIGHTLY_TAG} - OK" + fi +fi + +echo "Checking API/ABI compatibility..." +set +e +bash "$SCRIPT_DIR/check-release-apiabi.sh" +APIABI_RESULT=$? +set -e +if [[ $APIABI_RESULT -ne 0 ]]; then + if [[ "$DRY_RUN" == "true" ]]; then + echo "Warning: API/ABI check found backwards-incompatible changes (dry run, continuing)." + CHECKS_PASSED=false + else + echo "Error: API/ABI check found backwards-incompatible changes." + exit 1 + fi +else + echo "API/ABI compatibility check passed - OK" +fi + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then echo "checks_passed=${CHECKS_PASSED}" >> "$GITHUB_OUTPUT" fi diff --git a/scripts/make-release-summary.txt b/scripts/make-release-summary.txt index 38da80df16fa..8e64a509903b 100644 --- a/scripts/make-release-summary.txt +++ b/scripts/make-release-summary.txt @@ -5,7 +5,11 @@ Write a summary of the change log in a few sections: ``` ## Overview -[an overview using 1 to 3 sentences (no line breaks)] +[a single paragraph overview of all changes] + +### Highlights (if applicable) + +[go through all Pull Requests and write short summaries of those that have the "highlight" label] ### API changes (if applicable) @@ -39,9 +43,11 @@ Write a summary of the change log in a few sections: Guidelines: -- All bullet point in the summary should be concise and rarely exceed a single line of 120 characters +- All bullet point in the summary should be concise and rarely exceed a single line of 120 characters (excluding PR links) - Avoid repeating `ggml`-specific changes - these should already be covered by the `ggml` release links - Provide PR link for each bullet point where possible - Don't add bullet point to state that there are no API changes in some module +- Combine related topics (a full list of commits will be appended independently at end of the summary) +- Skip minor-impact notes (e.g. "fix compile warnings", "refactored code", ...) -Output just the summary in a markdown block, without any extra text. +Output just the summary in a markdown block, without any extra text. Save it to a local text file called `release-notes-vX.Y.Z.txt`. diff --git a/scripts/snapdragon/ggml-hexagon-align-macros.py b/scripts/snapdragon/ggml-hexagon-align-macros.py new file mode 100755 index 000000000000..b64db3e654a8 --- /dev/null +++ b/scripts/snapdragon/ggml-hexagon-align-macros.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +""" +align-macros.py - Inspect and align trailing backslashes in multiline C/C++ macros. + +Usage: + align-macros.py [paths...] # Check and report misaligned macros + align-macros.py --diff [paths...] # Show unified diff of fixes + align-macros.py --fix [paths...] # Fix misaligned macros in-place + align-macros.py --fix --mode majority ... # Align to the dominant column + align-macros.py --fix --pad 2 ... # Align to (max_content_len + pad) + +Safety rules: + - Macros that are ALREADY aligned are NEVER touched (unless --all is given). + - Whitespace after trailing backslashes is flagged and cleaned. +""" + +import argparse +import difflib +import logging +import os +import re +import sys +from collections import Counter +from typing import List, Optional, Tuple, NamedTuple + +logger = logging.getLogger("ggml-hexagon-align-macros") + + +class MacroLine(NamedTuple): + line_num: int # 1-indexed + raw: str # Original line including newline + content: str # Line content before trailing backslash (stripped of trailing whitespace) + bs_col: Optional[int] # 1-indexed column of backslash, or None if last line has no backslash + trailing_ws: bool # True if whitespace existed after the backslash + + +class MacroDef(NamedTuple): + name: str + filepath: str + start_line: int + end_line: int + lines: List[MacroLine] + + +def parse_macros(filepath: str) -> List[MacroDef]: + """Extract all multiline macros from a C/C++ source file.""" + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + lines = f.readlines() + except Exception as e: + logger.error(f"Error reading {filepath}: {e}") + return [] + + macros: List[MacroDef] = [] + i = 0 + n = len(lines) + + while i < n: + line = lines[i] + m = re.match(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z0-9_]*)", line) + if m: + macro_name = m.group(1) + macro_start = i + 1 + macro_lines: List[MacroLine] = [] + cur = i + + while cur < n: + l_raw = lines[cur] + l_rstrip = l_raw.rstrip("\r\n") + + # Check if line has a trailing backslash + # Note: handle possible accidental spaces after backslash + match_bs = re.search(r"\\([ \t]*)$", l_rstrip) + if match_bs: + has_trailing_ws = len(match_bs.group(1)) > 0 + bs_index = match_bs.start() + content = l_rstrip[:bs_index].rstrip() + # 1-indexed column of the backslash + bs_col = bs_index + 1 + macro_lines.append(MacroLine( + line_num=cur + 1, + raw=l_raw, + content=content, + bs_col=bs_col, + trailing_ws=has_trailing_ws + )) + cur += 1 + else: + # Line does not end with backslash + if cur == i: + # Single-line macro, not multiline + break + else: + # Final line of a multiline macro + macro_lines.append(MacroLine( + line_num=cur + 1, + raw=l_raw, + content=l_rstrip.rstrip(), + bs_col=None, + trailing_ws=False + )) + break + + # Only record if it is a multiline macro (has at least one continuation line) + continuation_lines = [ml for ml in macro_lines if ml.bs_col is not None] + if continuation_lines: + macro_end = macro_lines[-1].line_num + macros.append(MacroDef( + name=macro_name, + filepath=filepath, + start_line=macro_start, + end_line=macro_end, + lines=macro_lines + )) + i = cur + i += 1 + + return macros + + +def is_macro_aligned(macro: MacroDef) -> bool: + """A macro is aligned if all continuation lines have backslashes at the same column.""" + bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None] + if not bs_cols: + return True + has_trailing_ws = any(ml.trailing_ws for ml in macro.lines) + return len(set(bs_cols)) == 1 and not has_trailing_ws + + +def compute_target_column(macro: MacroDef, mode: str, pad: int, target_col: Optional[int]) -> int: + """Determine the column where backslashes should be aligned.""" + max_content_len = max(len(ml.content) for ml in macro.lines) + min_needed = max_content_len + pad + + if target_col is not None: + return max(target_col, min_needed) + + bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None] + if not bs_cols: + return min_needed + + if mode == "min": + return min_needed + elif mode == "max": + return max(max(bs_cols), min_needed) + elif mode == "majority": + counts = Counter(bs_cols) + # Sort by frequency descending, then by column descending + majority_col = sorted(counts.items(), key=lambda x: (-x[1], -x[0]))[0][0] + return max(majority_col, min_needed) + else: + return min_needed + + +def realign_macro_lines(macro: MacroDef, target_col: int) -> List[str]: + """Format macro lines with backslashes aligned at target_col.""" + new_lines: List[str] = [] + for ml in macro.lines: + nl = "\r\n" if ml.raw.endswith("\r\n") else "\n" + if ml.bs_col is None: + # Last line without backslash + new_lines.append(ml.raw) + else: + if not ml.content: + spaces = " " * (target_col - 1) + new_lines.append(f"{spaces}\\{nl}") + else: + spaces_needed = max(1, target_col - len(ml.content) - 1) + new_lines.append(f"{ml.content}{' ' * spaces_needed}\\{nl}") + return new_lines + + +def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Optional[str]]: + macros = parse_macros(filepath) + if not macros: + return 0, 0, None + + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + file_lines = f.readlines() + + misaligned_count = 0 + modified = False + new_file_lines = list(file_lines) + + for macro in macros: + aligned = is_macro_aligned(macro) + if not aligned or args.all: + if not aligned: + misaligned_count += 1 + + bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None] + max_content = max(len(ml.content) for ml in macro.lines) + col_counts = Counter(bs_cols) + + if not args.quiet: + logger.info(f"{filepath}:{macro.start_line}-{macro.end_line} [{macro.name}]") + logger.info(f" Max content width: {max_content}, Min needed column (+{args.pad}): {max_content + args.pad}") + logger.info(f" Current backslash columns: {dict(sorted(col_counts.items()))}") + trailing_ws_lines = [ml.line_num for ml in macro.lines if ml.trailing_ws] + if trailing_ws_lines: + logger.warning(f" Warning: Trailing whitespace after backslash on line(s): {trailing_ws_lines}") + + target_col = compute_target_column(macro, args.mode, args.pad, args.target_col) + if not args.quiet: + logger.info(f" -> Target alignment column: {target_col}") + + realigned = realign_macro_lines(macro, target_col) + + start_idx = macro.start_line - 1 + end_idx = start_idx + len(macro.lines) + if new_file_lines[start_idx:end_idx] != realigned: + new_file_lines[start_idx:end_idx] = realigned + modified = True + + diff_text = None + if modified: + diff = difflib.unified_diff( + file_lines, + new_file_lines, + fromfile=f"a/{filepath}", + tofile=f"b/{filepath}", + lineterm="" + ) + diff_text = "\n".join(diff) + + if args.fix: + with open(filepath, "w", encoding="utf-8") as f: + f.writelines(new_file_lines) + if not args.quiet: + logger.info(f" [FIXED] Updated {filepath}") + + return len(macros), misaligned_count, diff_text + + +def find_source_files(paths: List[str]) -> List[str]: + extensions = {".c", ".cpp", ".cc", ".cxx", ".h", ".hpp", ".inl"} + result: List[str] = [] + for p in paths: + if os.path.isfile(p): + result.append(p) + elif os.path.isdir(p): + for root, _, files in os.walk(p): + for file in sorted(files): + _, ext = os.path.splitext(file) + if ext.lower() in extensions: + result.append(os.path.join(root, file)) + return sorted(result) + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(message)s") + parser = argparse.ArgumentParser( + description="Inspect and align backslashes in multiline C/C++ macros." + ) + parser.add_argument("paths", nargs="*", default=["."], help="Files or directories to scan (default: current dir)") + parser.add_argument("--fix", action="store_true", help="Fix misaligned macros in-place") + parser.add_argument("--diff", action="store_true", help="Display unified diff of suggested fixes") + parser.add_argument("--check", action="store_true", help="Exit with code 1 if misaligned macros exist") + parser.add_argument("--mode", choices=["min", "max", "majority"], default="min", + help="Alignment mode: 'min' (max_len + pad), 'max' (max existing col), 'majority' (dominant col)") + parser.add_argument("--pad", type=int, default=2, help="Spaces between longest line and backslash (default: 2)") + parser.add_argument("--target-col", type=int, default=None, help="Force alignment to an exact column") + parser.add_argument("--all", action="store_true", help="Realign all macros even if already aligned (default: only misaligned)") + parser.add_argument("-q", "--quiet", action="store_true", help="Only output errors and diffs/summary") + + args = parser.parse_args() + + files = find_source_files(args.paths) + if not files: + logger.error("No C/C++ source files found.") + sys.exit(0) + + total_macros = 0 + total_misaligned = 0 + diffs: List[str] = [] + + for filepath in files: + num_macros, num_misaligned, diff_text = process_file(filepath, args) + total_macros += num_macros + total_misaligned += num_misaligned + if diff_text: + diffs.append(diff_text) + + if args.diff and diffs: + logger.info("\n--- Proposed Changes ---\n") + for d in diffs: + logger.info(d) + + logger.info(f"\nSummary: scanned {len(files)} files, {total_macros} multiline macros, {total_misaligned} misaligned.") + + if args.check and total_misaligned > 0: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/snapdragon/ggml-hexagon-inspect.py b/scripts/snapdragon/ggml-hexagon-inspect.py new file mode 100755 index 000000000000..c977f6a17a24 --- /dev/null +++ b/scripts/snapdragon/ggml-hexagon-inspect.py @@ -0,0 +1,1388 @@ +#!/usr/bin/env python3 +""" +ggml-hexagon-inspect.py - Hexagon DSP binary inspection and diagnostic tool. + +Inspects Hexagon ELF binaries (libggml-htp-v*.so) for: + - Register spills (--spills): counts scalar and HVX vector stack spills, + separating in-loop spills from frame setup/teardown. + - Function disassembly (--disasm <func>): annotated disassembly showing + hardware loop bounds, packet boundaries, and spill instructions. + - Crash address resolution (--addr2line <addr...>): maps hex crash offsets + to function symbols, offsets, and source lines. + - CI verification (--strict): fails with non-zero exit if in-loop vector + spills or DMA worker vector instructions are detected. + +Usage: + # Check spills across all functions or specific operations + ./scripts/snapdragon/ggml-hexagon-inspect.py --spills + ./scripts/snapdragon/ggml-hexagon-inspect.py --spills --func "^compute_" + ./scripts/snapdragon/ggml-hexagon-inspect.py --spills --func "^compute_" --strict + + # Disassemble a function with annotated loop and spill markers + ./scripts/snapdragon/ggml-hexagon-inspect.py --disasm compute_same_shape_div_f32 + + # Resolve crash addresses (CLI arguments or piped logcat/FARF logs) + ./scripts/snapdragon/ggml-hexagon-inspect.py --addr2line 0x51a30 0x5ba54 + adb logcat | ./scripts/snapdragon/ggml-hexagon-inspect.py --addr2line +""" + +import argparse +import logging +import os +import platform +import re +import shutil +import signal +import subprocess +import sys +from pathlib import Path +from typing import Dict, List, NamedTuple, Optional, Set, Tuple + +# Ignore SIGPIPE to handle pipes (e.g. head, grep) gracefully +if hasattr(signal, "SIGPIPE"): + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) +logger = logging.getLogger("ggml-hexagon-inspect") + + +class InsnInfo(NamedTuple): + address: int + asm_text: str + is_vec: bool + is_vspill: bool + is_sspill: bool + is_store: bool + is_load: bool + in_loop: bool + + +class LoopStats: + def __init__(self, loop_type: str, start_addr: int, end_addr: Optional[int] = None, loop_id: int = 0): + self.loop_id = loop_id + self.loop_type = loop_type # "loop0" or "loop1" + self.start_addr = start_addr + self.end_addr = end_addr + self.packet_count = 0 + self.insn_count = 0 + self.vec_insn_count = 0 + self.vspills_st = 0 + self.vspills_ld = 0 + self.sspills_st = 0 + self.sspills_ld = 0 + + @property + def vspills_total(self) -> int: + return self.vspills_st + self.vspills_ld + + @property + def sspills_total(self) -> int: + return self.sspills_st + self.sspills_ld + + @property + def has_v_roundtrip(self) -> bool: + return self.vspills_st > 0 and self.vspills_ld > 0 + + @property + def vec_density(self) -> float: + return (self.vec_insn_count / self.packet_count) if self.packet_count > 0 else 0.0 + + +class FuncStats: + def __init__(self, name: str, address: int, size: int): + self.name = name + self.address = address + self.size = size + self.packet_count = 0 + self.insn_count = 0 + self.vec_insn_count = 0 + self.loop_count = 0 + self.vspills_in_loop = 0 + self.vspills_in_loop_st = 0 + self.vspills_in_loop_ld = 0 + self.vspills_total = 0 + self.sspills_in_loop = 0 + self.sspills_in_loop_st = 0 + self.sspills_in_loop_ld = 0 + self.sspills_total = 0 + self.promotions_in_loop = 0 + self.promotions_total = 0 + self.promotion_targets: Dict[str, int] = {} + self.calls_in_loop = 0 + self.calls_total = 0 + self.loops: List[LoopStats] = [] + self.insns: List[InsnInfo] = [] + + +class SymbolEntry(NamedTuple): + address: int + size: int + name: str + + +# Regular expression patterns for Hexagon disassembly parsing +RE_SYMBOL_HEADER = re.compile(r"^([0-9a-fA-F]+)\s+<([^>]+)>:", re.MULTILINE) +RE_INSN_LINE = re.compile( + r"^\s*([0-9a-fA-F]+):\s+([0-9a-fA-F]{2}(?:\s+[0-9a-fA-F]{2}){3})\s+([0-9a-fA-F]{8})\s*(.*)$" +) +RE_LOOP0_START = re.compile(r"\bloop0\((0x[0-9a-fA-F]+)") +RE_LOOP1_START = re.compile(r"\bloop1\((0x[0-9a-fA-F]+)") +RE_VMEM_BASE = re.compile(r"\bvmemu?\s*\(\s*([a-z0-9]+)\b") +RE_SMEM_BASE = re.compile(r"\bmem[bwhd](?:_locked|_fifo)?\s*\(\s*([a-z0-9]+)\b") +RE_MEM_STORE = re.compile(r"\bv?mem[bwhdu]?(?:_[a-z]+)?\s*\([^)]*\)\s*(\+|-)?=") +RE_ADD_OP = re.compile(r"\b(r[0-9]+)\s*=\s*add\s*\(\s*([^,()]+)\s*,\s*([^,()]+)\s*\)") +RE_ASSIGN_LHS = re.compile(r"^\s*(?:if\s*\([^)]+\)\s*)?(r[0-9]+)(?::(r[0-9]+))?\s*(?:[+\-*/&|^]?=)") +RE_VEC_OP = re.compile(r"\b(v[0-9]+|w[0-9]+|q[0-3]|vmemu?)\b") +RE_PROMOTION_CALL = re.compile( + r"\b(?:call|jump)\s+(?:0x[0-9a-fA-F]+\s+)?<(__(?:trunc|extend)[a-zA-Z0-9_]+)(?:@plt)?>" +) +RE_ANY_CALL = re.compile(r"\bcallr?\b") + + +def is_mem_store(insn: str) -> bool: + return bool(RE_MEM_STORE.search(insn)) + + +def update_sp_regs(insn: str, sp_regs: Set[str]) -> None: + # Track registers derived from stack frame (r29/r30) + m_add = RE_ADD_OP.search(insn) + if m_add: + dest = m_add.group(1) + op1 = m_add.group(2).strip() + op2 = m_add.group(3).strip() + if op1 in sp_regs or op2 in sp_regs: + sp_regs.add(dest) + return + + m_assign = RE_ASSIGN_LHS.match(insn.strip()) + if m_assign: + r1 = m_assign.group(1) + r2 = m_assign.group(2) + if r1 and r1 not in ("r29", "r30"): + sp_regs.discard(r1) + if r2 and r2 not in ("r29", "r30"): + sp_regs.discard(r2) + + +def get_repo_root() -> Path: + # Resolve repository root from script location + return Path(__file__).resolve().parent.parent.parent + + +def extract_arch_num(p: Path) -> int: + # Extract integer architecture version (e.g. v81 -> 81) + m = re.search(r"-v([0-9]+)\.so$", p.name) + return int(m.group(1)) if m else 0 + + +def find_default_lib(repo_root: Path, arch_filter: Optional[str] = None) -> Optional[Path]: + # Search for built Hexagon shared libraries in build and pkg directories + candidates = [] + search_dirs = [ + repo_root / "build-adb" / "ggml" / "src" / "ggml-hexagon", + repo_root / "build-android" / "ggml" / "src" / "ggml-hexagon", + repo_root / "build-ubuntu" / "ggml" / "src" / "ggml-hexagon", + repo_root / "build-linux" / "ggml" / "src" / "ggml-hexagon", + repo_root / "pkg-adb" / "llama.cpp" / "lib", + repo_root / "pkg-android" / "llama.cpp" / "lib", + repo_root / "pkg-ubuntu" / "llama.cpp" / "lib", + ] + + arch_needle = None + if arch_filter: + arch_needle = arch_filter if arch_filter.startswith("v") else f"v{arch_filter}" + + for d in search_dirs: + if not d.is_dir(): + continue + for p in d.glob("libggml-htp-*.so"): + if arch_needle and arch_needle not in p.name: + continue + candidates.append(p) + + if not candidates: + for p in repo_root.glob("build-*/ggml/src/ggml-hexagon/libggml-htp-*.so"): + if arch_needle and arch_needle not in p.name: + continue + candidates.append(p) + + if not candidates: + return None + + # Group latest build candidates (within 60s of max mtime) and pick highest arch + max_mtime = max(p.stat().st_mtime for p in candidates) + recent = [p for p in candidates if max_mtime - p.stat().st_mtime <= 60] + recent.sort(key=lambda p: extract_arch_num(p), reverse=True) + return recent[0] + + +def translate_container_arg(arg: str, repo_root: Path) -> str: + # Do not translate non-path command flags + if arg.startswith("-") and "=" not in arg: + return arg + if arg.startswith("--") and "=" in arg: + flag, val = arg.split("=", 1) + return f"{flag}={translate_container_arg(val, repo_root)}" + try: + p = Path(arg) + if (p.is_absolute() and p.exists()) or (p.exists() and ("/" in arg or "\\" in arg)): + resolved = p.resolve() + if resolved.is_relative_to(repo_root): + rel = resolved.relative_to(repo_root) + return f"/workspace/{rel.as_posix()}" + except Exception: + pass + return arg + + +class HexagonToolchain: + def __init__( + self, + repo_root: Path, + use_docker: bool = False, + image_url: str = "ghcr.io/snapdragon-toolchain", + image_name: str = "arm64-android", + image_ver: str = "v0.7", + ): + self.repo_root = repo_root + self.image = f"{image_url}/{image_name}:{image_ver}" + self.docker_bin = shutil.which("docker") + self.use_docker = use_docker + + if not use_docker: + self.native_objdump, self.native_addr2line = self._discover_native_tools() + else: + self.native_objdump = None + self.native_addr2line = None + + if not self.native_objdump and not self.native_addr2line: + self.use_docker = True + + def _discover_native_tools(self) -> Tuple[Optional[str], Optional[str]]: + # Check system PATH + objdump = shutil.which("hexagon-llvm-objdump") + addr2line = shutil.which("hexagon-addr2line") or shutil.which("hexagon-llvm-addr2line") + + # Check HEXAGON_TOOLS_ROOT environment variable + tools_root = os.environ.get("HEXAGON_TOOLS_ROOT") + if tools_root: + bin_dir = Path(tools_root) / "Tools" / "bin" + objdump_path = bin_dir / "hexagon-llvm-objdump" + addr2line_path = bin_dir / "hexagon-addr2line" + if objdump_path.is_file() and not objdump: + objdump = str(objdump_path) + if addr2line_path.is_file() and not addr2line: + addr2line = str(addr2line_path) + + # Check HEXAGON_SDK_ROOT environment variable + sdk_root = os.environ.get("HEXAGON_SDK_ROOT") + if sdk_root: + tools_parent = Path(sdk_root) / "tools" / "HEXAGON_Tools" + if tools_parent.is_dir(): + for t_dir in tools_parent.iterdir(): + bin_dir = t_dir / "Tools" / "bin" + objdump_path = bin_dir / "hexagon-llvm-objdump" + addr2line_path = bin_dir / "hexagon-addr2line" + if objdump_path.is_file() and not objdump: + objdump = str(objdump_path) + if addr2line_path.is_file() and not addr2line: + addr2line = str(addr2line_path) + + return objdump, addr2line + + def run_tool(self, tool_name: str, args: List[str], stdin_data: Optional[str] = None) -> str: + # Execute tool either natively or inside Docker container + if not self.use_docker: + tool_path = self.native_objdump if "objdump" in tool_name else self.native_addr2line + if not tool_path: + tool_path = shutil.which(tool_name) + if not tool_path: + raise RuntimeError(f"Tool {tool_name} not found natively. Use Docker instead.") + + cmd = [tool_path] + args + res = subprocess.run(cmd, capture_output=True, text=True, input=stdin_data) + if res.returncode != 0: + raise RuntimeError(f"Tool {tool_name} failed: {res.stderr.strip()}") + return res.stdout + + # Running via Docker container + if not self.docker_bin: + raise RuntimeError("Docker is required but not installed or found on PATH.") + + container_tools_dir = "/opt/hexagon/6.6.0.0/tools/HEXAGON_Tools/19.0.07/Tools/bin" + if "objdump" in tool_name: + container_tool = f"{container_tools_dir}/hexagon-llvm-objdump" + elif "addr2line" in tool_name: + container_tool = f"{container_tools_dir}/hexagon-addr2line" + elif "nm" in tool_name: + container_tool = f"{container_tools_dir}/llvm-nm" + else: + container_tool = f"{container_tools_dir}/{tool_name}" + + # Translate file paths from host to /workspace + translated_args = [translate_container_arg(arg, self.repo_root) for arg in args] + + docker_cmd = [ + "docker", + "run", + "--rm", + "--platform", + "linux/amd64", + "-v", + f"{self.repo_root}:/workspace", + "-w", + "/workspace", + ] + + if platform.system() != "Windows": + docker_cmd += ["-u", f"{os.getuid()}:{os.getgid()}"] + + docker_cmd += [self.image, container_tool] + translated_args + + res = subprocess.run(docker_cmd, capture_output=True, text=True, input=stdin_data) + if res.returncode != 0: + raise RuntimeError(f"Docker command failed: {res.stderr.strip()}") + return res.stdout + + +def parse_symbols(toolchain: HexagonToolchain, lib_path: Path) -> List[SymbolEntry]: + # Parse function symbols from library using objdump -t + output = toolchain.run_tool("hexagon-llvm-objdump", ["-t", str(lib_path)]) + sym_re = re.compile(r"^([0-9a-fA-F]+)\s+[lgw! ]+\s+F\s+\.text\s+([0-9a-fA-F]+)\s+(.+)$") + + symbols = [] + for line in output.splitlines(): + m = sym_re.match(line.strip()) + if m: + addr = int(m.group(1), 16) + size = int(m.group(2), 16) + name = m.group(3).strip() + symbols.append(SymbolEntry(addr, size, name)) + + symbols.sort(key=lambda s: s.address) + return symbols + + +def find_enclosing_symbol(symbols: List[SymbolEntry], address: int) -> Optional[Tuple[str, int]]: + # Binary search enclosing function symbol and compute offset + low = 0 + high = len(symbols) - 1 + best = None + + while low <= high: + mid = (low + high) // 2 + s = symbols[mid] + if s.address <= address: + if address < s.address + s.size: + return (s.name, address - s.address) + best = s + low = mid + 1 + else: + high = mid - 1 + + if best and address < best.address + best.size: + return (best.name, address - best.address) + return None + + +def parse_disassembly( + disasm_text: str, func_filter: Optional[re.Pattern] = None +) -> List[FuncStats]: + # Parse disassembly text into structured function statistics + matches = list(RE_SYMBOL_HEADER.finditer(disasm_text)) + funcs: List[FuncStats] = [] + + for i, m in enumerate(matches): + name = m.group(2) + if func_filter and not func_filter.search(name): + continue + + addr = int(m.group(1), 16) + start_idx = m.end() + end_idx = matches[i + 1].start() if i + 1 < len(matches) else len(disasm_text) + chunk = disasm_text[start_idx:end_idx] + + stats = FuncStats(name=name, address=addr, size=0) + + loop0_target: Optional[int] = None + loop1_target: Optional[int] = None + loop0_active = False + loop1_active = False + current_loop0: Optional[LoopStats] = None + current_loop1: Optional[LoopStats] = None + sp_regs: Set[str] = {"r29", "r30"} + + first_addr = None + last_addr = None + + for raw_line in chunk.splitlines(): + lm = RE_INSN_LINE.match(raw_line) + if not lm: + continue + + cur_addr = int(lm.group(1), 16) + asm_chunk = lm.group(4) + + if first_addr is None: + first_addr = cur_addr + last_addr = cur_addr + + # Track packet count + if "{" in asm_chunk: + stats.packet_count += 1 + if current_loop0: + current_loop0.packet_count += 1 + if current_loop1: + current_loop1.packet_count += 1 + + # Check loop starts + m0 = RE_LOOP0_START.search(asm_chunk) + if m0: + loop0_target = int(m0.group(1), 16) + stats.loop_count += 1 + + m1 = RE_LOOP1_START.search(asm_chunk) + if m1: + loop1_target = int(m1.group(1), 16) + stats.loop_count += 1 + + if loop0_target is not None and cur_addr >= loop0_target: + loop0_active = True + if current_loop0 is None: + current_loop0 = LoopStats( + loop_id=len(stats.loops) + 1, + loop_type="loop0", + start_addr=loop0_target, + end_addr=0, + ) + + if loop1_target is not None and cur_addr >= loop1_target: + loop1_active = True + if current_loop1 is None: + current_loop1 = LoopStats( + loop_id=len(stats.loops) + 1, + loop_type="loop1", + start_addr=loop1_target, + end_addr=0, + ) + + in_loop = loop0_active or loop1_active + + # Parse instructions within packet line + cleaned = re.sub(r"[{}\s]|:endloop[01]", " ", asm_chunk) + sub_insns = [p.strip() for p in cleaned.split(";") if p.strip()] + + for insn in sub_insns: + update_sp_regs(insn, sp_regs) + + stats.insn_count += 1 + if current_loop0: + current_loop0.insn_count += 1 + if current_loop1: + current_loop1.insn_count += 1 + + is_vec = bool(RE_VEC_OP.search(insn)) + if is_vec: + stats.vec_insn_count += 1 + if current_loop0: + current_loop0.vec_insn_count += 1 + if current_loop1: + current_loop1.vec_insn_count += 1 + + vm = RE_VMEM_BASE.search(insn) + is_vspill = bool(vm and vm.group(1) in sp_regs) + + sm = RE_SMEM_BASE.search(insn) + is_sspill = bool(sm and sm.group(1) in sp_regs) + + is_store = False + is_load = False + if is_vspill or is_sspill: + is_store = is_mem_store(insn) + is_load = not is_store + + if is_vspill: + stats.vspills_total += 1 + if in_loop: + stats.vspills_in_loop += 1 + if is_store: + stats.vspills_in_loop_st += 1 + else: + stats.vspills_in_loop_ld += 1 + if current_loop0: + if is_store: + current_loop0.vspills_st += 1 + else: + current_loop0.vspills_ld += 1 + if current_loop1: + if is_store: + current_loop1.vspills_st += 1 + else: + current_loop1.vspills_ld += 1 + elif is_sspill: + stats.sspills_total += 1 + if in_loop: + stats.sspills_in_loop += 1 + if is_store: + stats.sspills_in_loop_st += 1 + else: + stats.sspills_in_loop_ld += 1 + if current_loop0: + if is_store: + current_loop0.sspills_st += 1 + else: + current_loop0.sspills_ld += 1 + if current_loop1: + if is_store: + current_loop1.sspills_st += 1 + else: + current_loop1.sspills_ld += 1 + + is_call = bool(RE_ANY_CALL.search(insn)) + prom_m = RE_PROMOTION_CALL.search(insn) + if is_call: + stats.calls_total += 1 + if in_loop: + stats.calls_in_loop += 1 + if prom_m: + stats.promotions_total += 1 + ptarget = prom_m.group(1) + stats.promotion_targets[ptarget] = stats.promotion_targets.get(ptarget, 0) + 1 + if in_loop: + stats.promotions_in_loop += 1 + + stats.insns.append( + InsnInfo( + address=cur_addr, + asm_text=insn, + is_vec=is_vec, + is_vspill=is_vspill, + is_sspill=is_sspill, + is_store=is_store, + is_load=is_load, + in_loop=in_loop, + ) + ) + + # Check loop ends + if ":endloop0" in asm_chunk: + loop0_active = False + loop0_target = None + if current_loop0: + current_loop0.end_addr = cur_addr + stats.loops.append(current_loop0) + current_loop0 = None + + if ":endloop1" in asm_chunk: + loop1_active = False + loop1_target = None + if current_loop1: + current_loop1.end_addr = cur_addr + stats.loops.append(current_loop1) + current_loop1 = None + + if current_loop0: + current_loop0.end_addr = last_addr or 0 + stats.loops.append(current_loop0) + if current_loop1: + current_loop1.end_addr = last_addr or 0 + stats.loops.append(current_loop1) + + stats.loops.sort(key=lambda lp: lp.start_addr) + for idx, loop in enumerate(stats.loops, 1): + loop.loop_id = idx + + if first_addr is not None and last_addr is not None: + stats.size = (last_addr - first_addr) + 4 + + funcs.append(stats) + + return funcs + + +def annotate_disasm_line( + raw_line: str, + loop0_target: Optional[int], + loop1_target: Optional[int], + loop0_active: bool, + loop1_active: bool, + use_color: bool = True, + sp_regs: Optional[Set[str]] = None, +) -> Tuple[str, Optional[int], Optional[int], bool, bool, bool]: + # Annotate disassembly line with spill and loop tags + lm = RE_INSN_LINE.match(raw_line) + if not lm: + return raw_line, loop0_target, loop1_target, loop0_active, loop1_active, False + + cur_addr = int(lm.group(1), 16) + asm_chunk = lm.group(4) + is_event = False + + if sp_regs is None: + sp_regs = {"r29", "r30"} + + # Check loop starts + m0 = RE_LOOP0_START.search(asm_chunk) + if m0: + loop0_target = int(m0.group(1), 16) + m1 = RE_LOOP1_START.search(asm_chunk) + if m1: + loop1_target = int(m1.group(1), 16) + + if loop0_target is not None and cur_addr >= loop0_target: + loop0_active = True + if loop1_target is not None and cur_addr >= loop1_target: + loop1_active = True + + in_loop = loop0_active or loop1_active + + tags = [] + if m0: + tags.append("[LOOP0-START]") + is_event = True + if m1: + tags.append("[LOOP1-START]") + is_event = True + + cleaned = re.sub(r"[{}\s]|:endloop[01]", " ", asm_chunk) + sub_insns = [p.strip() for p in cleaned.split(";") if p.strip()] + + for insn in sub_insns: + update_sp_regs(insn, sp_regs) + + for insn in sub_insns: + vm = RE_VMEM_BASE.search(insn) + if vm and vm.group(1) in sp_regs: + base = vm.group(1) + is_st = is_mem_store(insn) + op = "STORE" if is_st else "LOAD" + tgt = f"({base})" if base not in ("r29", "r30") else "" + if in_loop: + tag = f"[V-SPILL:{op}{tgt}:IN-LOOP]" + tags.append(f"\033[1;31m{tag}\033[0m" if use_color else tag) + else: + tag = f"[V-SPILL:{op}{tgt}]" + tags.append(f"\033[1;33m{tag}\033[0m" if use_color else tag) + is_event = True + + sm = RE_SMEM_BASE.search(insn) + if sm and sm.group(1) in sp_regs: + base = sm.group(1) + is_st = is_mem_store(insn) + op = "STORE" if is_st else "LOAD" + tgt = f"({base})" if base not in ("r29", "r30") else "" + if in_loop: + tag = f"[S-SPILL:{op}{tgt}:IN-LOOP]" + tags.append(f"\033[1;35m{tag}\033[0m" if use_color else tag) + else: + tag = f"[S-SPILL:{op}{tgt}]" + tags.append(f"\033[0;35m{tag}\033[0m" if use_color else tag) + is_event = True + + prom_m = RE_PROMOTION_CALL.search(asm_chunk) + if prom_m: + ptarget = prom_m.group(1) + if in_loop: + tag = f"[PROMOTION:{ptarget}:IN-LOOP]" + tags.append(f"\033[1;31m{tag}\033[0m" if use_color else tag) + else: + tag = f"[PROMOTION:{ptarget}]" + tags.append(f"\033[1;35m{tag}\033[0m" if use_color else tag) + is_event = True + elif RE_ANY_CALL.search(asm_chunk): + if in_loop: + tag = "[CALL:IN-LOOP]" + tags.append(f"\033[1;31m{tag}\033[0m" if use_color else tag) + is_event = True + else: + tag = "[CALL]" + tags.append(f"\033[1;36m{tag}\033[0m" if use_color else tag) + + if ":endloop0" in asm_chunk: + tags.append("[LOOP0-END]") + loop0_active = False + loop0_target = None + is_event = True + if ":endloop1" in asm_chunk: + tags.append("[LOOP1-END]") + loop1_active = False + loop1_target = None + is_event = True + + tag_str = " ".join(tags) + if tag_str: + annotated = f"{raw_line:<80} {tag_str}" + else: + annotated = raw_line + + return annotated, loop0_target, loop1_target, loop0_active, loop1_active, is_event + + +def run_spills( + toolchain: HexagonToolchain, + lib_path: Path, + args: argparse.Namespace, +) -> int: + # Scan and report register spills across binary functions + logger.info(f"Inspecting library: {lib_path}") + disasm_text = toolchain.run_tool("hexagon-llvm-objdump", ["-d", str(lib_path)]) + + func_re = re.compile(args.func) if args.func else None + funcs = parse_disassembly(disasm_text, func_re) + + # Filter functions + reported = [] + for f in funcs: + has_spills = f.vspills_total > 0 or f.sspills_in_loop > 0 or f.sspills_total > 0 + if args.all or args.func or has_spills: + reported.append(f) + + # Sort: in-loop vector spills desc, then total vector spills desc, then in-loop scalar spills desc + reported.sort( + key=lambda x: (x.vspills_in_loop, x.vspills_total, x.sspills_in_loop, x.sspills_total), + reverse=True, + ) + + use_color = not args.no_color and sys.stdout.isatty() + + # Print summary table + col_addr = "Address" + col_name = "Function" + col_pkts = "Packets" + col_insn = "Insns" + col_vec = "HVX Ops" + col_vloop = "V-Loop (st/ld)" + col_vtot = "V-Tot" + col_sloop = "S-Loop (st/ld)" + col_stot = "S-Tot" + col_notes = "Notes" + + hdr = ( + f"{col_addr:<10} | {col_name:<40} | {col_pkts:>7} | {col_insn:>6} | " + f"{col_vec:>7} | {col_vloop:>14} | {col_vtot:>5} | {col_sloop:>14} | {col_stot:>5} | {col_notes}" + ) + sep = "-" * len(hdr) + + logger.info("\n" + sep) + logger.info(hdr) + logger.info(sep) + + tot_vloop = 0 + tot_sloop = 0 + tot_funcs_with_vloop = 0 + strict_violations = [] + + dma_re: Optional[re.Pattern[str]] = re.compile(args.dma_pattern) if args.dma_pattern else None + + for f in reported: + tot_vloop += f.vspills_in_loop + tot_sloop += f.sspills_in_loop + if f.vspills_in_loop > 0: + tot_funcs_with_vloop += 1 + + # Check strict criteria + if args.strict: + inloop_v = f.vspills_in_loop_st if getattr(args, "strict_stores_only", False) else f.vspills_in_loop + if inloop_v > args.max_inloop_vspills: + lbl = "in-loop vector store spills" if getattr(args, "strict_stores_only", False) else "in-loop vector spills" + strict_violations.append( + f"{f.name}: {inloop_v} {lbl} (max allowed: {args.max_inloop_vspills})" + ) + if dma_re and dma_re.search(f.name): + if f.vec_insn_count > args.max_dma_vec_ops: + strict_violations.append( + f"{f.name}: DMA worker contains {f.vec_insn_count} HVX vector ops (max allowed: {args.max_dma_vec_ops})" + ) + + vloop_detail = f"{f.vspills_in_loop} ({f.vspills_in_loop_st}s,{f.vspills_in_loop_ld}l)" if f.vspills_in_loop > 0 else "0" + sloop_detail = f"{f.sspills_in_loop} ({f.sspills_in_loop_st}s,{f.sspills_in_loop_ld}l)" if f.sspills_in_loop > 0 else "0" + + notes = "" + if f.vspills_in_loop_st > 0 and f.vspills_in_loop_ld > 0: + notes = "\033[1;31m[V-ROUNDTRIP!]\033[0m" if use_color else "[V-ROUNDTRIP!]" + elif f.vspills_in_loop_st == 0 and f.vspills_in_loop_ld > 0: + notes = "v-readonly" + + vloop_str = f"{vloop_detail:>14}" + if f.vspills_in_loop > 0 and use_color: + if f.vspills_in_loop_st > 0 and f.vspills_in_loop_ld > 0: + vloop_str = f"\033[1;31m{vloop_str}\033[0m" + else: + vloop_str = f"\033[1;33m{vloop_str}\033[0m" + + sloop_str = f"{sloop_detail:>14}" + + logger.info( + f"0x{f.address:08x} | {f.name:<40} | {f.packet_count:>7} | {f.insn_count:>6} | " + f"{f.vec_insn_count:>7} | {vloop_str} | {f.vspills_total:>5} | {sloop_str} | {f.sspills_total:>5} | {notes}" + ) + + logger.info(sep) + logger.info( + f"Total functions analyzed: {len(funcs)} | Reported: {len(reported)} | " + f"Functions with in-loop vector spills: {tot_funcs_with_vloop} | " + f"Total in-loop vector spills: {tot_vloop} | Total in-loop scalar spills: {tot_sloop}" + ) + + if args.strict: + logger.info("\n" + "=" * 50) + if strict_violations: + if use_color: + logger.error("\033[1;31mSTRICT CHECK FAILED\033[0m") + else: + logger.error("STRICT CHECK FAILED") + for v in strict_violations: + logger.error(f" - {v}") + logger.info("=" * 50) + return 1 + else: + if use_color: + logger.info("\033[1;32mSTRICT CHECK PASSED: 0 violations\033[0m") + else: + logger.info("STRICT CHECK PASSED: 0 violations") + logger.info("=" * 50) + + return 0 + + +def run_promotions( + toolchain: HexagonToolchain, + lib_path: Path, + args: argparse.Namespace, +) -> int: + # Scan and report soft-float promotion calls across binary functions + logger.info(f"Inspecting library: {lib_path}") + disasm_text = toolchain.run_tool("hexagon-llvm-objdump", ["-d", str(lib_path)]) + + func_re = re.compile(args.func) if args.func else None + funcs = parse_disassembly(disasm_text, func_re) + + reported = [] + for f in funcs: + if args.all or f.promotions_total > 0: + reported.append(f) + + # Sort: in-loop promotions desc, then total promotions desc + reported.sort( + key=lambda x: (x.promotions_in_loop, x.promotions_total), + reverse=True, + ) + + use_color = not args.no_color and sys.stdout.isatty() + + col_addr = "Address" + col_name = "Function" + col_loop = "Loops" + col_inloop = "In-Loop" + col_tot = "Total" + col_targets = "Promotion Targets" + + hdr = f"{col_addr:<10} | {col_name:<44} | {col_loop:>5} | {col_inloop:>7} | {col_tot:>5} | {col_targets}" + sep = "-" * max(len(hdr), 110) + + logger.info("\n" + sep) + logger.info(hdr) + logger.info(sep) + + tot_inloop = 0 + tot_prom = 0 + tot_funcs_with_prom = 0 + strict_violations = [] + + for f in reported: + tot_inloop += f.promotions_in_loop + tot_prom += f.promotions_total + if f.promotions_total > 0: + tot_funcs_with_prom += 1 + + if args.strict: + max_p = args.max_promotions if args.max_promotions is not None else 0 + if f.promotions_total > max_p: + strict_violations.append( + f"{f.name}: {f.promotions_total} float promotion calls (max allowed: {max_p})" + ) + + inloop_str = f"{f.promotions_in_loop:>7}" + if f.promotions_in_loop > 0 and use_color: + inloop_str = f"\033[1;31m{inloop_str}\033[0m" + + targets_str = ", ".join(f"{t}: {c}" for t, c in sorted(f.promotion_targets.items())) + logger.info( + f"0x{f.address:08x} | {f.name:<44} | {f.loop_count:>5} | {inloop_str} | {f.promotions_total:>5} | {targets_str}" + ) + + logger.info(sep) + logger.info( + f"Total functions analyzed: {len(funcs)} | Reported: {len(reported)} | " + f"Functions with float promotions: {tot_funcs_with_prom} | " + f"Total promotion calls: {tot_prom} | In-loop: {tot_inloop}" + ) + + if args.strict: + logger.info("\n" + "=" * 50) + if strict_violations: + if use_color: + logger.error("\033[1;31mSTRICT CHECK FAILED\033[0m") + else: + logger.error("STRICT CHECK FAILED") + for v in strict_violations: + logger.error(f" - {v}") + logger.info("=" * 50) + return 1 + else: + if use_color: + logger.info("\033[1;32mSTRICT CHECK PASSED: 0 violations\033[0m") + else: + logger.info("STRICT CHECK PASSED: 0 violations") + logger.info("=" * 50) + + return 0 + + +def run_disasm( + toolchain: HexagonToolchain, + lib_path: Path, + args: argparse.Namespace, +) -> int: + # Disassemble matching function(s) with annotated loop and spill markers + func_pattern = args.disasm if args.disasm else (args.func or ".*") + logger.info(f"Inspecting library: {lib_path}") + logger.info(f"Disassembling functions matching: '{func_pattern}'\n") + + # Disassemble symbol + disasm_text = toolchain.run_tool( + "hexagon-llvm-objdump", + ["-d", f"--disassemble-symbols={func_pattern}", str(lib_path)], + ) + + # If --disassemble-symbols yielded nothing (e.g. pattern was a regex), dump whole binary and filter + matches = list(RE_SYMBOL_HEADER.finditer(disasm_text)) + if not matches: + all_disasm = toolchain.run_tool("hexagon-llvm-objdump", ["-d", str(lib_path)]) + pat = re.compile(func_pattern) + all_matches = list(RE_SYMBOL_HEADER.finditer(all_disasm)) + matched_symbols = [m.group(2) for m in all_matches if pat.search(m.group(2))] + if not matched_symbols: + logger.error(f"Error: No symbols found matching '{func_pattern}'.") + return 1 + # Re-run with symbol list bounded by limit + sym_limit = args.limit if hasattr(args, "limit") and args.limit and args.limit > 0 else len(matched_symbols) + sym_arg = ",".join(matched_symbols[:sym_limit]) + disasm_text = toolchain.run_tool( + "hexagon-llvm-objdump", + ["-d", f"--disassemble-symbols={sym_arg}", str(lib_path)], + ) + matches = list(RE_SYMBOL_HEADER.finditer(disasm_text)) + + use_color = not args.no_color and sys.stdout.isatty() + + # Parse and log annotated functions + for i, m in enumerate(matches): + name = m.group(2) + addr = int(m.group(1), 16) + start_idx = m.end() + end_idx = matches[i + 1].start() if i + 1 < len(matches) else len(disasm_text) + chunk = disasm_text[start_idx:end_idx] + + # Parse statistics for this function + func_stats = parse_disassembly(disasm_text[m.start():end_idx])[0] + + # Log header + hdr_border = "=" * 80 + logger.info(hdr_border) + logger.info(f"Function: {name}") + logger.info(f"Address: 0x{addr:08x} - 0x{addr + func_stats.size:08x} ({func_stats.size} bytes)") + logger.info(f"Packets: {func_stats.packet_count} | Instructions: {func_stats.insn_count} | Loops: {func_stats.loop_count}") + vec_pct = (func_stats.vec_insn_count / func_stats.insn_count * 100.0) if func_stats.insn_count else 0.0 + logger.info(f"HVX Ops: {func_stats.vec_insn_count} ({vec_pct:.1f}% of instructions)") + vloop_info = f"{func_stats.vspills_in_loop} ({func_stats.vspills_in_loop_st} st, {func_stats.vspills_in_loop_ld} ld)" + sloop_info = f"{func_stats.sspills_in_loop} ({func_stats.sspills_in_loop_st} st, {func_stats.sspills_in_loop_ld} ld)" + logger.info( + f"Spills: Vector in-loop: {vloop_info} | Vector total: {func_stats.vspills_total} | " + f"Scalar in-loop: {sloop_info} | Scalar total: {func_stats.sspills_total}" + ) + logger.info( + f"Calls: Total: {func_stats.calls_total} (in-loop: {func_stats.calls_in_loop}) | " + f"Float promotions: {func_stats.promotions_total} (in-loop: {func_stats.promotions_in_loop})" + ) + logger.info(hdr_border) + + # Print Loop Breakdown Table if function has loops + if func_stats.loops: + logger.info(f"\n--- Loops ({len(func_stats.loops)}) " + "-" * 67) + loop_hdr = ( + f"{'#':<3} | {'Type':<5} | {'Address Range':<25} | {'Packets':>7} | " + f"{'HVX Ops':>7} | {'Vec/Pkt':>7} | {'V-Spills (st, ld)':>17} | {'S-Spills (st, ld)':>17} | Notes" + ) + logger.info(loop_hdr) + logger.info("-" * len(loop_hdr)) + for loop in func_stats.loops: + vspill_str = f"{loop.vspills_total} ({loop.vspills_st}s,{loop.vspills_ld}l)" + sspill_str = f"{loop.sspills_total} ({loop.sspills_st}s,{loop.sspills_ld}l)" + notes = [] + if loop.has_v_roundtrip: + notes.append("\033[1;31m[V-ROUNDTRIP!]\033[0m" if use_color else "[V-ROUNDTRIP!]") + elif loop.vspills_st == 0 and loop.vspills_ld > 0: + notes.append("v-readonly") + if loop.vec_density >= 1.5: + notes.append("\033[1;32mdual-hvx\033[0m" if use_color else "dual-hvx") + notes_str = ", ".join(notes) + logger.info( + f"{loop.loop_id:<3} | {loop.loop_type:<5} | 0x{loop.start_addr:08x} - 0x{loop.end_addr:08x} | " + f"{loop.packet_count:>7} | {loop.vec_insn_count:>7} | {loop.vec_density:>7.2f} | " + f"{vspill_str:>17} | {sspill_str:>17} | {notes_str}" + ) + logger.info("-" * len(loop_hdr) + "\n") + + # Parse lines and annotations + lines = chunk.splitlines() + annotated_lines = [] + is_event_list = [] + loop0_target = None + loop1_target = None + loop0_active = False + loop1_active = False + sp_regs = {"r29", "r30"} + + for line in lines: + ann_line, loop0_target, loop1_target, loop0_active, loop1_active, is_ev = annotate_disasm_line( + line, loop0_target, loop1_target, loop0_active, loop1_active, use_color, sp_regs + ) + annotated_lines.append(ann_line) + is_event_list.append(is_ev) + + # Filter output if --spills-only + if getattr(args, "spills_only", False): + ctx = args.context if args.context is not None else 2 + to_show = [False] * len(annotated_lines) + for idx, ev in enumerate(is_event_list): + if ev: + for j in range(max(0, idx - ctx), min(len(annotated_lines), idx + ctx + 1)): + to_show[j] = True + + if not any(to_show): + logger.info(" (No spills, promotions, or in-loop calls detected in this function)\n") + else: + in_gap = False + for idx, show in enumerate(to_show): + if show: + in_gap = False + logger.info(annotated_lines[idx]) + else: + if not in_gap: + logger.info(" ...") + in_gap = True + logger.info("") + else: + for ann_line in annotated_lines: + logger.info(ann_line) + logger.info("") + + return 0 + + +def extract_addresses_from_input(lines: List[str]) -> List[int]: + # Extract hex program counter addresses from input lines + re_pc = re.compile(r"\b(?:pc|PC|ip|IP)\s*(?:=|:|\s)\s*0*(?:0x)?([0-9a-fA-F]{3,8})\b") + re_plus_hex = re.compile(r"\+0x([0-9a-fA-F]{3,8})\b") + re_hex = re.compile(r"\b0x([0-9a-fA-F]{3,8})\b") + re_bare_hex = re.compile(r"^\s*0*([0-9a-fA-F]{3,8})\s*$") + + addrs = [] + seen = set() + + for line in lines: + matched = False + for m in re_pc.finditer(line): + val = int(m.group(1), 16) + if val not in seen: + seen.add(val) + addrs.append(val) + matched = True + + if not matched: + for m in re_plus_hex.finditer(line): + val = int(m.group(1), 16) + if val not in seen: + seen.add(val) + addrs.append(val) + matched = True + + if not matched: + for m in re_hex.finditer(line): + val = int(m.group(1), 16) + if val not in seen: + seen.add(val) + addrs.append(val) + matched = True + + if not matched: + m = re_bare_hex.match(line) + if m: + val = int(m.group(1), 16) + if val not in seen: + seen.add(val) + addrs.append(val) + + return addrs + + +def run_addr2line( + toolchain: HexagonToolchain, + lib_path: Path, + args: argparse.Namespace, +) -> int: + # Resolve addresses or crash logs to source locations and symbols + input_addrs: List[int] = [] + + if args.addr2line: + for arg in args.addr2line: + if arg == "-": + continue + try: + val = int(arg, 16) + input_addrs.append(val) + except ValueError: + # Treat as text line and search for hex addresses + input_addrs.extend(extract_addresses_from_input([arg])) + + # Read from stdin if piped or requested via '-' + if not sys.stdin.isatty() or "-" in (args.addr2line or []): + stdin_lines = sys.stdin.readlines() + input_addrs.extend(extract_addresses_from_input(stdin_lines)) + + if not input_addrs: + logger.error("Error: No addresses found to resolve. Provide hex addresses or pipe crash logs to stdin.") + logger.error("Example: ./scripts/snapdragon/ggml-hexagon-inspect.py --addr2line 0x51a30 0x5ba54") + return 1 + + logger.info(f"Resolving {len(input_addrs)} address(es) against: {lib_path}\n") + + # Load symbol table for symbol + offset fallback + symbols = parse_symbols(toolchain, lib_path) + + # Format addresses for addr2line tool (prefixed with 0x) + addr_strs = [f"0x{a:x}" for a in input_addrs] + tool_args = ["-e", str(lib_path), "-f", "-C", "-p", "-a"] + addr_strs + + raw_output = toolchain.run_tool("hexagon-addr2line", tool_args) + + # Parse output lines + # Format: 0x51a30: binary_thread_add_id_f32 at /path/file.c:123 + re_out = re.compile(r"^(0x[0-9a-fA-F]+):\s+(.*?)\s+at\s+(.*)$") + + for line in raw_output.splitlines(): + line = line.strip() + if not line: + continue + m = re_out.match(line) + if m: + addr_hex = m.group(1) + addr_val = int(addr_hex, 16) + func_name = m.group(2) + src_loc = m.group(3) + + # Check if function name is unknown or generic, look up symbol table + sym_info = find_enclosing_symbol(symbols, addr_val) + if sym_info: + sym_name, sym_offset = sym_info + sym_display = f"{sym_name}+0x{sym_offset:x}" + else: + sym_display = func_name + + logger.info(f"{addr_hex:<12} -> {sym_display:<40} ({src_loc})") + else: + logger.info(line) + + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Inspect Hexagon DSP binaries for register spills, function disassembly, and crash analysis." + ) + + # Target library + parser.add_argument( + "--lib", + help="Path to Hexagon shared library (e.g. libggml-htp-v81.so). Auto-detected if omitted.", + ) + parser.add_argument( + "--arch", + help="Architecture version filter for auto-detection (e.g. v75, v79, v81).", + ) + + # Modes + parser.add_argument( + "--spills", + action="store_true", + help="Scan binary and report scalar/vector stack spills table.", + ) + parser.add_argument( + "--promotions", + action="store_true", + help="Scan binary and report functions with soft-float promotion calls (__trunc*, __extend*).", + ) + parser.add_argument( + "--disasm", + nargs="?", + const="", + metavar="FUNC", + help="Disassemble function symbol or regex pattern with annotated loop and spill markers.", + ) + parser.add_argument( + "--spills-only", + action="store_true", + help="In --disasm, only display packets containing spills, promotions, or in-loop calls, with surrounding context.", + ) + parser.add_argument( + "-C", + "--context", + type=int, + default=None, + metavar="N", + help="Number of context packets before and after spills in --disasm --spills-only (default: 2).", + ) + parser.add_argument( + "--limit", + type=int, + default=20, + help="Maximum symbols to disassemble when using pattern in --disasm (default: 20, 0 for unlimited).", + ) + parser.add_argument( + "--addr2line", + nargs="*", + metavar="ADDR", + help="Resolve hex addresses or piped crash traces to symbols and source lines.", + ) + + # Filtering & Display + parser.add_argument( + "--func", + "--fn", + "-f", + help="Regex filter for function names in --spills, --promotions, or --disasm.", + ) + parser.add_argument( + "--all", + "-a", + action="store_true", + help="Show all functions in table, even those with 0 spills/promotions.", + ) + parser.add_argument( + "--no-color", + action="store_true", + help="Disable ANSI color output.", + ) + + # Strict check options + parser.add_argument( + "--strict", + action="store_true", + help="CI mode: exit with non-zero status if violations (in-loop vector spills, DMA worker vector ops) occur.", + ) + parser.add_argument( + "--max-inloop-vspills", + type=int, + default=0, + help="Maximum allowed in-loop vector spills in --strict mode (default: 0).", + ) + parser.add_argument( + "--strict-stores-only", + action="store_true", + help="In --strict mode, only count vector store spills (st > 0) towards violations, ignoring readonly stack loads.", + ) + parser.add_argument( + "--max-dma-vec-ops", + type=int, + default=0, + help="Maximum allowed vector instructions in DMA workers in --strict mode (default: 0).", + ) + parser.add_argument( + "--max-promotions", + type=int, + default=None, + help="Maximum allowed float promotion calls in --strict mode (default: 0).", + ) + parser.add_argument( + "--dma-pattern", + default=r"^.*_thread(?:_.*)?$", + help="Regex pattern identifying DMA worker functions (default: '^.*_thread(?:_.*)?$').", + ) + + # Toolchain options + parser.add_argument( + "--docker", + action="store_true", + help="Force execution inside Docker container.", + ) + parser.add_argument( + "--no-docker", + action="store_true", + help="Force native execution on host instead of Docker.", + ) + parser.add_argument( + "--toolchain-version", + default="v0.7", + help="Docker toolchain tag (default: v0.7).", + ) + parser.add_argument( + "--toolchain-url", + default="ghcr.io/snapdragon-toolchain", + help="Docker toolchain registry (default: ghcr.io/snapdragon-toolchain).", + ) + parser.add_argument( + "--image-name", + default="arm64-android", + help="Docker toolchain image name (default: arm64-android).", + ) + + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) + + repo_root = get_repo_root() + + # Determine target library + lib_path = None + if args.lib: + lib_path = Path(args.lib).resolve() + if not lib_path.is_file(): + logger.error(f"Error: Specified library '{args.lib}' does not exist.") + sys.exit(1) + else: + lib_path = find_default_lib(repo_root, args.arch) + if not lib_path: + logger.error("Error: No Hexagon library found in build-* or pkg-* directories.") + logger.error("Build the project first via ./scripts/snapdragon/build.py --target adb or specify --lib.") + sys.exit(1) + + # Initialize toolchain wrapper + use_docker = args.docker or (not args.no_docker and platform.system() == "Darwin") + try: + toolchain = HexagonToolchain( + repo_root=repo_root, + use_docker=use_docker, + image_url=args.toolchain_url, + image_name=args.image_name, + image_ver=args.toolchain_version, + ) + except Exception as e: + logger.error(f"Error initializing toolchain: {e}") + sys.exit(1) + + # Dispatch commands + if args.addr2line is not None: + sys.exit(run_addr2line(toolchain, lib_path, args)) + elif args.disasm is not None: + sys.exit(run_disasm(toolchain, lib_path, args)) + elif args.promotions: + sys.exit(run_promotions(toolchain, lib_path, args)) + else: + # Default action is --spills + sys.exit(run_spills(toolchain, lib_path, args)) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) + main() diff --git a/scripts/snapdragon/ggml-hexagon-profile.py b/scripts/snapdragon/ggml-hexagon-profile.py index 038d92fb5c47..4ac227678dcb 100755 --- a/scripts/snapdragon/ggml-hexagon-profile.py +++ b/scripts/snapdragon/ggml-hexagon-profile.py @@ -7,7 +7,7 @@ import statistics import logging import bisect -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Iterable from collections import defaultdict @@ -54,6 +54,7 @@ def device_matches(record_device, target_device): return False +logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) logger = logging.getLogger("ggml-hexagon-profile") @@ -473,6 +474,8 @@ def print_bubbles_timeline(op): all_bubbles = [] for t in active_threads: stats = thread_stats[t] + assert isinstance(stats['dma_bubbles'], Iterable) + assert isinstance(stats['compute_bubbles'], Iterable) for start, end, dur in stats['compute_bubbles']: pct = (dur / batch_duration) * 100.0 all_bubbles.append((dur, f"Thread {t} Compute: bubble of {dur} cycles ({pct:.1f}%) at {start - op_start} to {end - op_start}")) @@ -646,7 +649,7 @@ def main(): args = parser.parse_args() - logging.basicConfig(level=logging.INFO, format='%(message)s') + logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) if "pmu" in args.sort and args.pmu_index is None: logger.error(f"Cannot sort by '{args.sort}' without --pmu-index.") diff --git a/scripts/snapdragon/ggml-hexagon-trace.py b/scripts/snapdragon/ggml-hexagon-trace.py index 99bf771b85d8..760eb57d9c10 100755 --- a/scripts/snapdragon/ggml-hexagon-trace.py +++ b/scripts/snapdragon/ggml-hexagon-trace.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional from collections import defaultdict +logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) logger = logging.getLogger("ggml-hexagon-trace") op_pattern = re.compile( @@ -732,7 +733,7 @@ def main(): group.add_argument("--tail", type=int, help="Limit to last N ops") args = parser.parse_args() - logging.basicConfig(level=logging.INFO, format='%(message)s') + logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) op_filter_re = None if args.filter: diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.py index 81eecd2e0ceb..6d845c341722 100755 --- a/scripts/snapdragon/run.py +++ b/scripts/snapdragon/run.py @@ -14,6 +14,44 @@ logger = logging.getLogger("run") +MANAGED_ENV_NAMES = ( + "GGML_HEXAGON_DEVICES", + "GGML_HEXAGON_VERBOSE", + "GGML_HEXAGON_PROFILE", + "GGML_HEXAGON_NHVX", + "GGML_HEXAGON_NHMX", + "GGML_HEXAGON_HOSTBUF", + "GGML_HEXAGON_DMA64", + "GGML_HEXAGON_OPBATCH", + "GGML_HEXAGON_OPQUEUE", + "GGML_HEXAGON_OPPOLL", + "GGML_HEXAGON_OPFILTER", + "GGML_HEXAGON_OPFUSION", + "GGML_HEXAGON_VMEM", + "GGML_HEXAGON_MBUF", + "GGML_HEXAGON_MM_SELECT", + "GGML_HEXAGON_FA_SELECT", + "GGML_HEXAGON_GDN_SELECT", + "GGML_HEXAGON_AR_SELECT", + "GGML_HEXAGON_ETM", + "GGML_HEXAGON_ARCH", + "GGML_HEXAGON_OPTRACE", + "GGML_OPENCL_PLATFORM", + "GGML_OPENCL_DEVICE", + "GGML_OPENCL_OPFILTER", + "GGML_OPENCL_KERNEL_CACHE_DIR", + "GGML_OPENCL_KERNEL_CACHE_DEBUG", + "GGML_OPENCL_FA_TUNE", + "GGML_OPENCL_DISABLE_FUSION", + "GGML_OPENCL_ADRENO_XMEM_GEMM", + "GGML_OPENCL_ADRENO_USE_LARGE_BUFFER", + "GGML_SCHED_DEBUG", + "MTMD_BACKEND_DEVICE", + "D", + "DEVICE", +) + + def parse_target(target_str): if not target_str: return None, None @@ -38,6 +76,57 @@ def shlex_join(args_list): return " ".join(pipes.quote(x) for x in args_list) +def split_device_list(devices): + parts = [] + curr = [] + bracket_depth = 0 + + for ch in devices: + if ch == '[': + bracket_depth += 1 + curr.append(ch) + elif ch == ']': + if bracket_depth > 0: + bracket_depth -= 1 + curr.append(ch) + elif ch == ',' and bracket_depth == 0: + part = "".join(curr).strip() + if part: + parts.append(part) + curr = [] + else: + curr.append(ch) + + part = "".join(curr).strip() + if part: + parts.append(part) + + return parts + + +def device_arg_from_devices(devices): + if devices.isdigit(): + n = int(devices) + return ",".join(f"HTP{i}" for i in range(n)) + + names = [] + for part in split_device_list(devices): + if "[" in part: + part = part.split("[", 1)[0].strip() + if part: + names.append(part) + + return ",".join(names) + + +def normalize_cmd_device_args(cmd_args): + for i, arg in enumerate(cmd_args): + if arg == "--device" and i + 1 < len(cmd_args): + cmd_args[i + 1] = device_arg_from_devices(cmd_args[i + 1]) + elif arg.startswith("--device="): + cmd_args[i] = "--device=" + device_arg_from_devices(arg.split("=", 1)[1]) + + def main(): logging.basicConfig(level=logging.INFO, format='%(message)s') # Split arguments at '--' @@ -68,6 +157,7 @@ def main(): parser.add_argument("--hex-nhvx", help="Number of HVX units to use (GGML_HEXAGON_NHVX)") parser.add_argument("--hex-nhmx", help="Number of HMX units to use. 0 disables HMX power-up (GGML_HEXAGON_NHMX)") parser.add_argument("--hex-hostbuf", help="Enable host buffers (GGML_HEXAGON_HOSTBUF)") + parser.add_argument("--hex-dma64", nargs="?", const="1", help="Enable (1) or disable (0) 64-bit DMA for model weights (GGML_HEXAGON_DMA64)") parser.add_argument("--hex-opbatch", help="Maximum number of operations to batch into a single HTP execution (GGML_HEXAGON_OPBATCH)") parser.add_argument("--hex-opqueue", help="Size of the asynchronous NPU operation queue (GGML_HEXAGON_OPQUEUE)") parser.add_argument("--hex-oppoll", default="1", help="Enable (1) or Disable (0) polling for NPU opbatch completion (GGML_HEXAGON_OPPOLL) (default: 1)") @@ -75,8 +165,9 @@ def main(): parser.add_argument("--hex-opfusion", help="NPU graph node fusion optimization level (0: disabled, 1: enabled) (GGML_HEXAGON_OPFUSION)") parser.add_argument("--hex-vmem", help="Maximum NPU VMEM size limit in MB to allocate (GGML_HEXAGON_VMEM)") parser.add_argument("--hex-mbuf", help="Maximum host buffer size limit in MB to allocate (GGML_HEXAGON_MBUF)") - parser.add_argument("--hex-mm-select", help="Select MUL_MAT and MUL_MAT_ID kernel (GGML_HEXAGON_MM_SELECT) 3:HMX,2:HVX-tiled,1:HVX-flat,0:disable") + parser.add_argument("--hex-mm-select", help="Select MUL_MAT and MUL_MAT_ID kernel (GGML_HEXAGON_MM_SELECT) 2:HMX,1:HVX,0:disable") parser.add_argument("--hex-fa-select", help="Select Flash Attention kernel (GGML_HEXAGON_FA_SELECT) 2:HMX,1:HVX,0:disable") + parser.add_argument("--hex-gdn-select", help="Select Gated Delta Net kernel (GGML_HEXAGON_GDN_SELECT) 2:HMX,1:HVX,0:disable") parser.add_argument("--hex-ar-select", help="Select All-Reduce kernel (GGML_HEXAGON_AR_SELECT) 1:enable,0:disable") parser.add_argument("--hex-etm", help="Enable Embedded Trace Macrocell hardware tracing / trace logging (GGML_HEXAGON_ETM)") parser.add_argument("--hex-arch", help="Target Hexagon NPU architecture version override (v73, v75, v79, v81, etc.) (GGML_HEXAGON_ARCH)") @@ -142,8 +233,6 @@ def main(): def set_env(env_name, opt_val): if opt_val is not None: env_vars[env_name] = str(opt_val) - elif env_name in os.environ: - env_vars[env_name] = os.environ[env_name] # Resolve and filter devices (HTP vs OpenCL) device_in_cmd = None @@ -166,7 +255,7 @@ def set_env(env_name, opt_val): hex_devices = devices_val cl_device = "" else: - parts = [p.strip() for p in devices_val.split(",")] + parts = split_device_list(devices_val) # Any device containing "htp" is Hexagon, rest is OpenCL hex_parts = [p for p in parts if "htp" in p.lower()] cl_parts = [ @@ -181,15 +270,13 @@ def set_env(env_name, opt_val): # Set Hexagon devices if hex_devices: env_vars["GGML_HEXAGON_DEVICES"] = hex_devices - elif "GGML_HEXAGON_DEVICES" in os.environ: - env_vars["GGML_HEXAGON_DEVICES"] = os.environ["GGML_HEXAGON_DEVICES"] + + normalize_cmd_device_args(cmd_args) # Set OpenCL device (unless overridden by --cl-device) final_cl_device = args.cl_device if args.cl_device is not None else cl_device if final_cl_device: env_vars["GGML_OPENCL_DEVICE"] = final_cl_device - elif "GGML_OPENCL_DEVICE" in os.environ: - env_vars["GGML_OPENCL_DEVICE"] = os.environ["GGML_OPENCL_DEVICE"] # Map shared & backend-specific parameters with correct overrides @@ -206,13 +293,12 @@ def set_env(env_name, opt_val): if args.cl_fa_tune or args.profile is not None: env_vars["GGML_OPENCL_FA_TUNE"] = "1" - elif "GGML_OPENCL_FA_TUNE" in os.environ: - env_vars["GGML_OPENCL_FA_TUNE"] = os.environ["GGML_OPENCL_FA_TUNE"] # Other Hexagon environment variables set_env("GGML_HEXAGON_NHVX", args.hex_nhvx) set_env("GGML_HEXAGON_NHMX", args.hex_nhmx) set_env("GGML_HEXAGON_HOSTBUF", args.hex_hostbuf) + set_env("GGML_HEXAGON_DMA64", args.hex_dma64) set_env("GGML_HEXAGON_OPBATCH", args.hex_opbatch) set_env("GGML_HEXAGON_OPQUEUE", args.hex_opqueue) set_env("GGML_HEXAGON_OPPOLL", args.hex_oppoll) @@ -222,6 +308,7 @@ def set_env(env_name, opt_val): set_env("GGML_HEXAGON_MBUF", args.hex_mbuf) set_env("GGML_HEXAGON_MM_SELECT", args.hex_mm_select) set_env("GGML_HEXAGON_FA_SELECT", args.hex_fa_select) + set_env("GGML_HEXAGON_GDN_SELECT", args.hex_gdn_select) set_env("GGML_HEXAGON_AR_SELECT", args.hex_ar_select) set_env("GGML_HEXAGON_ETM", args.hex_etm) set_env("GGML_HEXAGON_ARCH", args.hex_arch) @@ -235,18 +322,12 @@ def set_env(env_name, opt_val): if args.cl_disable_fusion: env_vars["GGML_OPENCL_DISABLE_FUSION"] = "1" - elif "GGML_OPENCL_DISABLE_FUSION" in os.environ: - env_vars["GGML_OPENCL_DISABLE_FUSION"] = os.environ["GGML_OPENCL_DISABLE_FUSION"] if args.cl_adreno_xmem: env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = "1" - elif "GGML_OPENCL_ADRENO_XMEM_GEMM" in os.environ: - env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = os.environ["GGML_OPENCL_ADRENO_XMEM_GEMM"] if args.cl_adreno_large_buffer: env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = "1" - elif "GGML_OPENCL_ADRENO_USE_LARGE_BUFFER" in os.environ: - env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = os.environ["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] if args.sched_debug: env_vars["GGML_SCHED_DEBUG"] = "2" @@ -288,15 +369,7 @@ def set_env(env_name, opt_val): has_b = any(arg == "-b" for arg in cmd_args) if not has_b: if args.devices: - if args.devices.isdigit(): - n = int(args.devices) - device_val = ",".join(f"HTP{i}" for i in range(n)) - else: - device_val = args.devices - elif "D" in os.environ: - device_val = os.environ["D"] - elif "DEVICE" in os.environ: - device_val = os.environ["DEVICE"] + device_val = device_arg_from_devices(args.devices) else: device_val = "HTP0" if device_val: @@ -305,17 +378,10 @@ def set_env(env_name, opt_val): has_device = any(arg.startswith("--device") for arg in cmd_args) if not has_device: if args.devices: - if args.devices.isdigit(): - n = int(args.devices) - device_val = ",".join(f"HTP{i}" for i in range(n)) - else: - device_val = args.devices - elif "D" in os.environ: - device_val = os.environ["D"] - elif "DEVICE" in os.environ: - device_val = os.environ["DEVICE"] + device_val = device_arg_from_devices(args.devices) else: device_val = "HTP0" + if device_val: cmd_args += ["--device", device_val] @@ -415,6 +481,8 @@ def set_env(env_name, opt_val): else: local_env["LD_LIBRARY_PATH"] = lib_dir + os.path.pathsep + local_env.get("LD_LIBRARY_PATH", "") + for k in MANAGED_ENV_NAMES: + local_env.pop(k, None) for k, v in env_vars.items(): local_env[k] = v diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 7b44a311abd0..8f4afc721b2b 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -e91ded11bdcd78c42f9c8d3978ff6686eb4c1226 +e565a8f4ce2e462c4973a24c51098dd3c81c0256 diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py index 0170b5b168f4..b320884bbeba 100755 --- a/scripts/sync_vendor.py +++ b/scripts/sync_vendor.py @@ -5,7 +5,7 @@ import sys import subprocess -HTTPLIB_VERSION = "refs/tags/v0.54.1" +HTTPLIB_VERSION = "refs/tags/v0.57.1" # used by examples/gguf-hash, these repos have no release tag, so we pin a commit XXHASH_COMMIT = "9f465f1ea932d6ad9a26cd77496311ffa544cd68" diff --git a/scripts/tool_bench.py b/scripts/tool_bench.py index d9f5583d4a56..fb7df10f3bc5 100755 --- a/scripts/tool_bench.py +++ b/scripts/tool_bench.py @@ -52,8 +52,8 @@ sys.path.insert(0, Path(__file__).parent.parent.as_posix()) if True: - from tools.server.tests.utils import ServerProcess - from tools.server.tests.unit.test_tool_call import do_test_calc_result, do_test_hello_world, do_test_weather + from tools.server.tests.utils import ServerProcess # ty: ignore[unresolved-import] + from tools.server.tests.unit.test_tool_call import do_test_calc_result, do_test_hello_world, do_test_weather # ty: ignore[unresolved-import] @contextmanager diff --git a/scripts/ui-assets.cmake b/scripts/ui-assets.cmake index 0c1c4de555a1..dec34ebda96c 100644 --- a/scripts/ui-assets.cmake +++ b/scripts/ui-assets.cmake @@ -15,16 +15,264 @@ set(HF_BUCKET "" CACHE STRING "Hugging Face bucket name") set(HF_VERSION "" CACHE STRING "Version to download (empty = resolve from git)") set(HF_ENABLED "" CACHE STRING "Whether to allow HF Bucket download (ON/OFF)") set(BUILD_UI "" CACHE STRING "Build UI via npm (ON/OFF)") -set(LLAMA_UI_EMBED "" CACHE STRING "Path to llama-ui-embed helper") set(LLAMA_UI_GZIP "" CACHE STRING "Apply gzip compress to assets to save bandwidth") set(DIST_DIR "${UI_BINARY_DIR}/dist") set(SRC_DIST_DIR "${UI_SOURCE_DIR}/dist") set(WORK_DIR "${UI_BINARY_DIR}/ui-src") set(STAMP_FILE "${UI_BINARY_DIR}/.ui-stamp") +set(EMBED_STAMP "${UI_BINARY_DIR}/.ui-embed.sha256") set(UI_CPP "${UI_BINARY_DIR}/ui.cpp") set(UI_H "${UI_BINARY_DIR}/ui.h") +function(mime_from_ext name out_var) + string(FIND "${name}" "." ext REVERSE) + if(ext GREATER -1) + string(SUBSTRING "${name}" ${ext} -1 ext_full) + string(SUBSTRING "${ext_full}" 1 -1 ext_str) + else() + set(ext_str "") + endif() + if(ext_str STREQUAL "html") + set(m "text/html; charset=utf-8") + elseif(ext_str STREQUAL "css") + set(m "text/css") + elseif(ext_str STREQUAL "js") + set(m "application/javascript") + elseif(ext_str STREQUAL "json") + set(m "application/json") + elseif(ext_str STREQUAL "webmanifest") + set(m "application/manifest+json") + elseif(ext_str STREQUAL "svg") + set(m "image/svg+xml") + elseif(ext_str STREQUAL "png") + set(m "image/png") + elseif(ext_str STREQUAL "jpg" OR ext_str STREQUAL "jpeg") + set(m "image/jpeg") + elseif(ext_str STREQUAL "ico") + set(m "image/x-icon") + elseif(ext_str STREQUAL "woff") + set(m "font/woff") + elseif(ext_str STREQUAL "woff2") + set(m "font/woff2") + else() + set(m "application/octet-stream") + endif() + set(${out_var} "${m}" PARENT_SCOPE) +endfunction() + +# Fail when a dist tree is present but is missing files the UI needs at +# runtime; catches truncated/stale asset trees early with a useful message. +function(ui_validate_assets files in_dir) + list(LENGTH files n_assets) + if(n_assets EQUAL 0) + return() + endif() + + set(found_index FALSE) + set(found_manifest FALSE) + set(found_sw FALSE) + set(found_build_json FALSE) + set(found_version_json FALSE) + set(found_bundle_js FALSE) + set(found_bundle_css FALSE) + set(found_workbox_js FALSE) + + foreach(f ${files}) + get_filename_component(base "${f}" NAME) + if(base STREQUAL "index.html") + set(found_index TRUE) + elseif(base STREQUAL "manifest.webmanifest") + set(found_manifest TRUE) + elseif(base STREQUAL "sw.js") + set(found_sw TRUE) + elseif(base STREQUAL "build.json") + set(found_build_json TRUE) + elseif(base STREQUAL "version.json") + set(found_version_json TRUE) + elseif(base MATCHES "^bundle.*\\.js$") + set(found_bundle_js TRUE) + elseif(base MATCHES "^bundle.*\\.css$") + set(found_bundle_css TRUE) + elseif(base MATCHES "^workbox.*\\.js$") + set(found_workbox_js TRUE) + endif() + endforeach() + + set(missing "") + if(NOT found_index) + list(APPEND missing "index.html") + endif() + if(NOT found_manifest) + list(APPEND missing "manifest.webmanifest") + endif() + if(NOT found_sw) + list(APPEND missing "sw.js") + endif() + if(NOT found_build_json) + list(APPEND missing "build.json") + endif() + if(NOT found_version_json) + list(APPEND missing "version.json") + endif() + if(NOT found_bundle_js) + list(APPEND missing "bundle[hash].js") + endif() + if(NOT found_bundle_css) + list(APPEND missing "bundle[hash].css") + endif() + if(NOT found_workbox_js) + list(APPEND missing "workbox[hash].js") + endif() + + if(missing) + set(listing "") + foreach(f ${files}) + string(APPEND listing " ${f}\n") + endforeach() + set(missing_list "") + foreach(m ${missing}) + string(APPEND missing_list " ${m}\n") + endforeach() + message(FATAL_ERROR + "UI: current asset files:\n${listing}" + "UI: missing required asset(s):\n${missing_list}" + "UI: hint: try cleaning your build directory: ${in_dir}") + endif() +endfunction() + +# Generate ui.cpp/ui.h embedding every file of ${dist_dir} (empty table when +# it has no index.html), gzip-compressed when LLAMA_UI_GZIP is enabled. +function(emit_files dist_dir) + set(UI_TEMPLATE_DIR "${LLAMA_SOURCE_DIR}/tools/ui") + + # Collect the asset list once and reuse it for the fingerprint, + # validation, compression and embedding. + set(assets "") + if(EXISTS "${dist_dir}/index.html") + file(GLOB_RECURSE assets + LIST_DIRECTORIES false + RELATIVE "${dist_dir}" + "${dist_dir}/*") + list(FILTER assets EXCLUDE REGEX "^_gzip/") + list(SORT assets) + endif() + + if(LLAMA_UI_GZIP AND NOT DEFINED ENV{SOURCE_DATE_EPOCH}) + # Zero the gzip header timestamp so identical inputs give identical + # bytes (and therefore stable ETags) on every machine. + set(ENV{SOURCE_DATE_EPOCH} 0) + endif() + + # Fingerprint of every input that determines ui.cpp/ui.h: compression + # settings, the asset tree (names + SHA-256) and this script + templates. + set(fp "${LLAMA_UI_GZIP}|$ENV{SOURCE_DATE_EPOCH}|${CMAKE_VERSION}\n") + foreach(f ${assets}) + file(SHA256 "${dist_dir}/${f}" h) + string(APPEND fp "${f} ${h}\n") + endforeach() + foreach(g + "${CMAKE_CURRENT_FUNCTION_LIST_FILE}" + "${UI_TEMPLATE_DIR}/ui.h.in" + "${UI_TEMPLATE_DIR}/ui.cpp.in") + file(SHA256 "${g}" h) + string(APPEND fp "gen ${h}\n") + endforeach() + string(SHA256 fingerprint "${fp}") + + if(EXISTS "${EMBED_STAMP}" AND EXISTS "${UI_CPP}" AND EXISTS "${UI_H}") + file(READ "${EMBED_STAMP}" fp_saved) + string(STRIP "${fp_saved}" fp_saved) + if(fp_saved STREQUAL "${fingerprint}") + message(STATUS "UI: assets unchanged, skipping embedding") + return() + endif() + endif() + + # Drop the old stamp up front so a crash mid-generation cannot leave + # outputs and stamp out of sync. + file(REMOVE "${EMBED_STAMP}") + + ui_validate_assets("${assets}" "${dist_dir}") + + set(embed_dir "${dist_dir}") + set(use_gzip FALSE) + + if(EXISTS "${dist_dir}/index.html") + if(EXISTS "${dist_dir}/_gzip") + # a _gzip tree inside dist_dir can only be a leftover from an + # older version of this script that staged it there + file(REMOVE_RECURSE "${dist_dir}/_gzip") + message(STATUS "UI: removed stale gzip tree ${dist_dir}/_gzip") + endif() + if(LLAMA_UI_GZIP) + # Compress every asset into a parallel _gzip/ tree under the build + # directory, served with Content-Encoding: gzip. + set(gzip_root "${UI_BINARY_DIR}/ui-gzip") + set(gzip_dir "${gzip_root}/_gzip") + file(REMOVE_RECURSE "${gzip_root}") + foreach(f IN LISTS assets) + get_filename_component(asset_path "${dist_dir}/${f}" REALPATH) + get_filename_component(dst_dir "${gzip_dir}/${f}" DIRECTORY) + file(MAKE_DIRECTORY "${dst_dir}") + file(ARCHIVE_CREATE + OUTPUT "${gzip_dir}/${f}" + PATHS "${asset_path}" + FORMAT raw + COMPRESSION GZip + ) + endforeach() + message(STATUS "UI: gzip compression applied (${gzip_dir})") + set(embed_dir "${gzip_dir}") + set(use_gzip TRUE) + endif() + endif() + + list(LENGTH assets n_assets) + + # Per-asset arrays and table rows go into the ui.h.in / ui.cpp.in templates; + # configure_file only rewrites on content change, avoiding needless recompiles. + set(ASSET_ARRAYS "") + set(ASSET_TABLE "") + set(idx 0) + + foreach(f IN LISTS assets) + file(READ "${embed_dir}/${f}" hex HEX) + if(hex STREQUAL "") + message(FATAL_ERROR "UI: empty file: ${embed_dir}/${f}") + endif() + + string(REGEX REPLACE "(..)" "0x\\1," bytes "${hex}") + file(SHA256 "${embed_dir}/${f}" etag) + mime_from_ext("${f}" mime) + + string(APPEND ASSET_ARRAYS + "static const unsigned char asset_${idx}[] = {${bytes}};\n") + + string(APPEND ASSET_TABLE + " { \"${f}\", asset_${idx}, sizeof(asset_${idx}), \"\\\"${etag}\\\"\", \"${mime}\" },\n") + + math(EXPR idx "${idx} + 1") + endforeach() + + set(LLAMA_UI_HAS_ASSETS 0) + if(n_assets GREATER 0) + set(LLAMA_UI_HAS_ASSETS 1) + endif() + set(N_ASSETS "${n_assets}") + set(USE_GZIP false) + if(use_gzip) + set(USE_GZIP true) + endif() + + configure_file("${UI_TEMPLATE_DIR}/ui.h.in" "${UI_H}" @ONLY) + configure_file("${UI_TEMPLATE_DIR}/ui.cpp.in" "${UI_CPP}" @ONLY) + + # Write the embed stamp last, after both generated files succeeded. + file(WRITE "${EMBED_STAMP}" "${fingerprint}") + message(STATUS "UI: embedded ${n_assets} assets") +endfunction() + function(npm_build_should_skip out_var) set(${out_var} FALSE PARENT_SCOPE) @@ -203,16 +451,8 @@ function(hf_download version out_var out_resolved) message(STATUS "UI: downloading from ${resolved}: ${base}/dist.tar.gz") - file(DOWNLOAD "${base}/dist.tar.gz?download=true" "${archive}" - STATUS status TIMEOUT 300 ${auth_headers} - ) - list(GET status 0 rc) - if(NOT rc EQUAL 0) - list(GET status 1 errmsg) - message(STATUS "UI: download dist.tar.gz from ${resolved} failed: ${errmsg}") - continue() - endif() - + # Fetch the checksum first: when the archive we already have matches + # it, the expensive download is skipped and only extraction repeats. file(DOWNLOAD "${base}/dist.tar.gz.sha256?download=true" "${archive}.sha256" STATUS status TIMEOUT 30 ${auth_headers} ) @@ -223,17 +463,44 @@ function(hf_download version out_var out_resolved) continue() endif() - # Validate sha256 checkums + # Validate the sha256 checksum: reject anything that is not a full + # 64-hex-digit digest before touching the archive. file(READ "${archive}.sha256" expected) string(REGEX MATCH "^[0-9a-fA-F]+" expected "${expected}") string(TOLOWER "${expected}" expected) - file(SHA256 "${archive}" actual) - if("${expected}" STREQUAL "" OR NOT "${actual}" STREQUAL "${expected}") - message(STATUS "UI: checksum mismatch for dist.tar.gz from ${resolved}") + string(LENGTH "${expected}" expected_len) + if(NOT expected_len EQUAL 64) + message(STATUS "UI: invalid checksum from ${resolved}") continue() endif() - # Clear DIST_DIR to remove stale files first + set(actual "") + if(EXISTS "${archive}") + file(SHA256 "${archive}" actual) + endif() + + if("${actual}" STREQUAL "${expected}") + message(STATUS "UI: local dist.tar.gz matches checksum from ${resolved}, skipping download") + else() + file(DOWNLOAD "${base}/dist.tar.gz?download=true" "${archive}" + STATUS status TIMEOUT 300 ${auth_headers} + ) + list(GET status 0 rc) + if(NOT rc EQUAL 0) + list(GET status 1 errmsg) + message(STATUS "UI: download dist.tar.gz from ${resolved} failed: ${errmsg}") + continue() + endif() + + file(SHA256 "${archive}" actual) + if(NOT "${actual}" STREQUAL "${expected}") + message(STATUS "UI: checksum mismatch for dist.tar.gz from ${resolved}") + continue() + endif() + endif() + + # Remove the stamp with the dist tree it describes, together. + file(REMOVE "${STAMP_FILE}") file(REMOVE_RECURSE "${DIST_DIR}") file(ARCHIVE_EXTRACT INPUT "${archive}" DESTINATION "${DIST_DIR}") @@ -250,48 +517,6 @@ function(hf_download version out_var out_resolved) endforeach() endfunction() -function(emit_files dist_dir) - # If gzip is requested, compress every asset into a parallel _gzip/ tree - # the structure stays the same; for ex: /abc/def --> /_gzip/abc/def - # embed.cpp will check for _gzip and will pick it up - if(LLAMA_UI_GZIP AND EXISTS "${dist_dir}/index.html") - find_program(GZIP_EXECUTABLE gzip) - if(NOT GZIP_EXECUTABLE) - message(WARNING "UI: LLAMA_UI_GZIP requested but gzip not found, embedding uncompressed") - else() - set(gzip_dir "${dist_dir}/_gzip") - file(REMOVE_RECURSE "${gzip_dir}") - file(GLOB_RECURSE all_files RELATIVE "${dist_dir}" "${dist_dir}/*") - foreach(f ${all_files}) - get_filename_component(dst_dir "${gzip_dir}/${f}" DIRECTORY) - file(MAKE_DIRECTORY "${dst_dir}") - execute_process( - COMMAND "${GZIP_EXECUTABLE}" -c "${dist_dir}/${f}" - OUTPUT_FILE "${gzip_dir}/${f}" - RESULT_VARIABLE gz_rc - ) - if(NOT gz_rc EQUAL 0) - message(FATAL_ERROR "UI: gzip failed for ${f}") - endif() - endforeach() - message(STATUS "UI: gzip compression applied (${gzip_dir})") - endif() - endif() - - set(args "${UI_CPP}" "${UI_H}") - if(EXISTS "${dist_dir}/index.html") - list(APPEND args "${dist_dir}") - endif() - - execute_process( - COMMAND "${LLAMA_UI_EMBED}" ${args} - RESULT_VARIABLE rc - ) - if(NOT rc EQUAL 0) - message(FATAL_ERROR "UI: llama-ui-embed failed (${rc})") - endif() -endfunction() - # --------------------------------------------------------------------------- # 1. Priority 1: pre-built assets supplied in tools/ui/dist # --------------------------------------------------------------------------- @@ -321,27 +546,27 @@ endif() if(NOT provisioned AND HF_ENABLED) resolve_version(VERSION) + # Stamp a successful HF download: records bucket + requested version and + # lets later steps distinguish downloaded assets from locally built ones. + set(stamp_key "${HF_BUCKET}|${VERSION}") + set(stamp_ok FALSE) - if(EXISTS "${STAMP_FILE}" AND NOT "${VERSION}" STREQUAL "") + if(EXISTS "${STAMP_FILE}" AND EXISTS "${DIST_DIR}/index.html" AND NOT "${VERSION}" STREQUAL "") file(READ "${STAMP_FILE}" stamped) string(STRIP "${stamped}" stamped) - if("${stamped}" STREQUAL "${VERSION}") + if(stamped STREQUAL "${stamp_key}") set(stamp_ok TRUE) endif() endif() - set(have_assets FALSE) - if(EXISTS "${DIST_DIR}/index.html") - set(have_assets TRUE) - endif() - if(stamp_ok AND have_assets) - message(STATUS "UI: HF stamp '${stamped}' matches version, skipping HF fetch") + if(stamp_ok) + message(STATUS "UI: HF stamp matches '${stamp_key}', skipping HF fetch") set(provisioned TRUE) else() hf_download("${VERSION}" HF_OK HF_RESOLVED) if(HF_OK) - file(WRITE "${STAMP_FILE}" "${HF_RESOLVED}") - message(STATUS "UI: HF download succeeded, stamp updated (${HF_RESOLVED})") + file(WRITE "${STAMP_FILE}" "${stamp_key}") + message(STATUS "UI: HF download succeeded, stamp updated (${stamp_key}, resolved: ${HF_RESOLVED})") set(provisioned TRUE) else() message(STATUS "UI: HF download failed") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 137d98f35dab..667c0573ddf6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -35,47 +35,51 @@ target_compile_features(llama-expert-dispatch PUBLIC cxx_std_17) target_link_libraries(llama-expert-dispatch PUBLIC ggml) set_target_properties(llama-expert-dispatch PROPERTIES POSITION_INDEPENDENT_CODE ON) +set(LLAMA_CORE_SOURCES + llama.cpp + llama-adapter.cpp + llama-arch.cpp + llama-batch.cpp + llama-chat.cpp + llama-context.cpp + llama-tp-lockstep.cpp + pipeline/pipe-tp-msg.cpp + llama-cparams.cpp + llama-grammar.cpp + llama-graph.cpp + llama-hparams.cpp + llama-impl.cpp + llama-io.cpp + llama-kv-cache.cpp + llama-kv-cache-iswa.cpp + llama-kv-cache-paged.cpp + llama-kv-cache-dsa.cpp + llama-kv-cache-dsa-iswa.cpp + llama-kv-cache-msa.cpp + llama-kv-cache-dsv4.cpp + llama-memory.cpp + llama-memory-hybrid.cpp + llama-memory-hybrid-iswa.cpp + llama-memory-hybrid-idx.cpp + llama-memory-recurrent.cpp + llama-ml8-registry.cpp + llama-mmap.cpp + llama-model-loader.cpp + llama-model-saver.cpp + llama-model.cpp + llama-pipeline.cpp + llama-pipeline-gguf.cpp + llama-weight-pager.cpp + llama-quant.cpp + llama-sampler.cpp + llama-vocab.cpp + unicode-data.cpp + unicode.cpp +) + add_library(llama ../include/llama.h - llama.cpp - llama-adapter.cpp - llama-arch.cpp - llama-batch.cpp - llama-chat.cpp - llama-context.cpp - llama-tp-lockstep.cpp - pipeline/pipe-tp-msg.cpp - llama-cparams.cpp - llama-grammar.cpp - llama-graph.cpp - llama-hparams.cpp - llama-impl.cpp - llama-io.cpp - llama-kv-cache.cpp - llama-kv-cache-iswa.cpp - llama-kv-cache-paged.cpp - llama-kv-cache-dsa.cpp - llama-kv-cache-dsa-iswa.cpp - llama-kv-cache-msa.cpp - llama-kv-cache-dsv4.cpp - llama-memory.cpp - llama-memory-hybrid.cpp - llama-memory-hybrid-iswa.cpp - llama-memory-hybrid-idx.cpp - llama-memory-recurrent.cpp - llama-ml8-registry.cpp - llama-mmap.cpp - llama-model-loader.cpp - llama-model-saver.cpp - llama-model.cpp - llama-pipeline.cpp - llama-pipeline-gguf.cpp - llama-weight-pager.cpp - llama-quant.cpp - llama-sampler.cpp - llama-vocab.cpp - unicode-data.cpp - unicode.cpp + ${LLAMA_CORE_SOURCES} unicode.h ${LLAMA_MODELS_SOURCES} ${LLAMA_WEIGHT_PAGER_SOURCES} @@ -86,8 +90,15 @@ set_target_properties(llama PROPERTIES VERSION ${LLAMA_VERSION_BASE} SOVERSION ${LLAMA_VERSION_MAJOR} MACHO_CURRENT_VERSION 0 # keep macOS linker from seeing oversized version number + UNITY_BUILD ON + UNITY_BUILD_BATCH_SIZE 16 ) +# exclude non-model sources from unity build +# fork: weight-pager and memory-tier sources are not unity-build safe either +set_source_files_properties(${LLAMA_CORE_SOURCES} ${LLAMA_WEIGHT_PAGER_SOURCES} ${LLAMA_MEMORY_TIER_SOURCES} ../include/llama.h unicode.h + PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) + configure_file(llama-version.h.in ${CMAKE_CURRENT_BINARY_DIR}/llama-version.h @ONLY) target_include_directories(llama PRIVATE . ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/src/llama-adapter.cpp b/src/llama-adapter.cpp index e6678a66d2a9..df3654d86d9f 100644 --- a/src/llama-adapter.cpp +++ b/src/llama-adapter.cpp @@ -6,6 +6,8 @@ #include <map> #include <cassert> +#include <cerrno> +#include <cstring> #include <sstream> #include <stdexcept> @@ -146,22 +148,23 @@ llama_adapter_lora_weight * llama_adapter_lora::get_weight(ggml_tensor * w) { return nullptr; } -static void llama_adapter_lora_init_impl(llama_model & model, const char * path_lora, llama_adapter_lora & adapter) { - LLAMA_LOG_INFO("%s: loading lora adapter from '%s' ...\n", __func__, path_lora); - +static void llama_adapter_lora_init_impl(llama_model & model, FILE * file, llama_adapter_lora & adapter) { ggml_context * ctx_init; gguf_init_params meta_gguf_params = { /* .no_alloc = */ true, /* .ctx = */ &ctx_init, }; - gguf_context_ptr ctx_gguf { gguf_init_from_file(path_lora, meta_gguf_params) }; + gguf_context_ptr ctx_gguf { gguf_init_from_file_ptr(file, meta_gguf_params) }; if (!ctx_gguf) { - throw std::runtime_error("failed to load lora adapter file from " + std::string(path_lora)); + throw std::runtime_error("failed to load lora adapter from file"); } ggml_context_ptr ctx { ctx_init }; + // must come after gguf_init_from_file_ptr, the llama_file constructor moves the file position + llama_file gguf_file(file); + // check metadata { const gguf_context * gguf_ctx = ctx_gguf.get(); @@ -393,7 +396,6 @@ static void llama_adapter_lora_init_impl(llama_model & model, const char * path_ // set tensor data { - llama_file gguf_file(path_lora, "rb"); std::vector<uint8_t> read_buf; auto set_tensor = [&](ggml_tensor * orig, ggml_tensor * dev) { const size_t offs = gguf_get_data_offset(ctx_gguf.get()) + gguf_get_tensor_offset(ctx_gguf.get(), gguf_find_tensor(ctx_gguf.get(), orig->name)); @@ -421,10 +423,30 @@ static void llama_adapter_lora_init_impl(llama_model & model, const char * path_ } llama_adapter_lora * llama_adapter_lora_init(llama_model * model, const char * path_lora) { + LLAMA_LOG_INFO("%s: loading lora adapter from '%s' ...\n", __func__, path_lora); + + FILE * file = ggml_fopen(path_lora, "rb"); + if (!file) { + LLAMA_LOG_ERROR("%s: failed to open '%s': %s\n", __func__, path_lora, strerror(errno)); + return nullptr; + } + + llama_adapter_lora * adapter = llama_adapter_lora_init_from_file_ptr(model, file); + fclose(file); + + return adapter; +} + +llama_adapter_lora * llama_adapter_lora_init_from_file_ptr(llama_model * model, FILE * file) { + if (!file) { + LLAMA_LOG_ERROR("%s: file is NULL\n", __func__); + return nullptr; + } + llama_adapter_lora * adapter = new llama_adapter_lora(model); try { - llama_adapter_lora_init_impl(*model, path_lora, *adapter); + llama_adapter_lora_init_impl(*model, file, *adapter); return adapter; } catch (const std::exception & err) { LLAMA_LOG_ERROR("%s: failed to apply lora adapter: %s\n", __func__, err.what()); diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 3f9b468e867a..42ab217d9f07 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -64,6 +64,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = { { LLM_ARCH_STARCODER2, "starcoder2" }, { LLM_ARCH_MAMBA, "mamba" }, { LLM_ARCH_MAMBA2, "mamba2" }, + { LLM_ARCH_MAPLE, "maple" }, { LLM_ARCH_JAMBA, "jamba" }, { LLM_ARCH_FALCON_H1, "falcon-h1" }, { LLM_ARCH_XVERSE, "xverse" }, @@ -137,6 +138,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = { { LLM_ARCH_GROVEMOE, "grovemoe" }, { LLM_ARCH_APERTUS, "apertus" }, { LLM_ARCH_MINIMAX_01, "minimax-01" }, + { LLM_ARCH_HRM_TEXT, "hrm_text" }, { LLM_ARCH_MINIMAX_M2, "minimax-m2" }, { LLM_ARCH_MINIMAX_M3, "minimax-m3" }, { LLM_ARCH_COGVLM, "cogvlm" }, @@ -149,6 +151,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = { { LLM_ARCH_PADDLEOCR, "paddleocr" }, { LLM_ARCH_MIMO2, "mimo2" }, { LLM_ARCH_STEP35, "step35" }, + { LLM_ARCH_SPARK2_5, "spark2_5" }, { LLM_ARCH_LLAMA_EMBED, "llama-embed" }, { LLM_ARCH_MAINCODER, "maincoder" }, { LLM_ARCH_KIMI_LINEAR, "kimi-linear" }, @@ -248,6 +251,10 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = { { LLM_KV_FULL_ATTENTION_INTERVAL, "%s.full_attention_interval" }, { LLM_KV_NUM_LOOPS, "%s.num_loops" }, { LLM_KV_SKIP_LOOP_FINAL_NORM, "%s.skip_loop_final_norm" }, + { LLM_KV_HRM_LAYERS_PER_STACK, "%s.hrm.layers_per_stack" }, + { LLM_KV_HRM_H_CYCLES, "%s.hrm.h_cycles" }, + { LLM_KV_HRM_L_CYCLES, "%s.hrm.l_cycles" }, + { LLM_KV_HRM_PREFIX_LM, "%s.hrm.prefix_lm" }, { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" }, { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" }, @@ -450,6 +457,7 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = { { LLM_TENSOR_OUTPUT_NORM_LFM2, "token_embd_norm" }, // fix for wrong tensor name { LLM_TENSOR_OUTPUT, "output" }, { LLM_TENSOR_ROPE_FREQS, "rope_freqs" }, + { LLM_TENSOR_HRM_Z_L_INIT, "hrm.z_l_init" }, { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, @@ -742,6 +750,7 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = { {LLM_TENSOR_TOKEN_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, {LLM_TENSOR_POS_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, {LLM_TENSOR_TOKEN_TYPES, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, + {LLM_TENSOR_HRM_Z_L_INIT, {LLM_TENSOR_LAYER_INPUT, GGML_OP_ADD}}, {LLM_TENSOR_TOKEN_EMBD_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, // do the norms on the first layer (not the input layer) {LLM_TENSOR_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, {LLM_TENSOR_CLS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, @@ -1142,6 +1151,7 @@ bool llm_arch_is_diffusion(const llm_arch & arch) { bool llm_arch_supports_rs_rollback(const llm_arch & arch) { switch (arch) { + case LLM_ARCH_KIMI_K3: case LLM_ARCH_QWEN35: case LLM_ARCH_QWEN35MOE: case LLM_ARCH_QWEN4EXP: diff --git a/src/llama-arch.h b/src/llama-arch.h index 8a50f9f28f89..bca4ea65ce00 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -69,6 +69,7 @@ enum llm_arch { LLM_ARCH_STARCODER2, LLM_ARCH_MAMBA, LLM_ARCH_MAMBA2, + LLM_ARCH_MAPLE, LLM_ARCH_JAMBA, LLM_ARCH_FALCON_H1, LLM_ARCH_XVERSE, @@ -150,6 +151,7 @@ enum llm_arch { LLM_ARCH_PADDLEOCR, LLM_ARCH_MIMO2, LLM_ARCH_STEP35, + LLM_ARCH_SPARK2_5, LLM_ARCH_LLAMA_EMBED, LLM_ARCH_MAINCODER, LLM_ARCH_KIMI_LINEAR, @@ -163,6 +165,7 @@ enum llm_arch { LLM_ARCH_QWEN3TTS, LLM_ARCH_POCKETTTS, LLM_ARCH_MINIMAX_01, + LLM_ARCH_HRM_TEXT, LLM_ARCH_UNKNOWN, }; @@ -253,6 +256,10 @@ enum llm_kv { LLM_KV_FULL_ATTENTION_INTERVAL, LLM_KV_NUM_LOOPS, LLM_KV_SKIP_LOOP_FINAL_NORM, + LLM_KV_HRM_LAYERS_PER_STACK, + LLM_KV_HRM_H_CYCLES, + LLM_KV_HRM_L_CYCLES, + LLM_KV_HRM_PREFIX_LM, LLM_KV_ATTENTION_HEAD_COUNT, LLM_KV_ATTENTION_HEAD_COUNT_KV, @@ -724,6 +731,7 @@ enum llm_tensor { LLM_TENSOR_NEXTN_HC_HEAD_SCALE, LLM_TENSOR_MASKED_EMBD_CENTROIDS, LLM_TENSOR_MASKED_EMBD_ORDERING, + LLM_TENSOR_HRM_Z_L_INIT, LLM_TENSOR_FC, LLM_TENSOR_D2T, LLM_TENSOR_DSPARK_MARKOV_W1, diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 6d3c988ac5d5..c2b3f3dfbfcf 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -41,6 +41,7 @@ #include <set> #include <stdexcept> #include <string> +#include <unordered_map> // WP ubatch-width hint into ggml-cuda's graph-capture prefill classifier // (defined in ggml-cuda.cu; weak so CPU/Vulkan-only builds link clean). @@ -1734,6 +1735,40 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3 } } +static int llama_graph_n_input_tensors(ggml_cgraph * gf) { + std::unordered_map<const ggml_tensor *, std::vector<ggml_tensor *>> users; + for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) { + ggml_tensor * node = ggml_graph_node(gf, i); + if (node->flags & GGML_TENSOR_FLAG_INPUT) { + users[node].push_back(node); + } + for (int j = 0; j < GGML_MAX_SRC; ++j) { + ggml_tensor * src = node->src[j]; + if (!src) { + break; + } + if (src->flags & GGML_TENSOR_FLAG_INPUT) { + users[src].push_back(node); + } + } + } + + for (const auto & [tensor, nodes] : users) { + if (tensor->op != GGML_OP_NONE) { + LLAMA_LOG_WARN("%s: input tensor '%32s' has op %s, expected GGML_OP_NONE\n", + __func__, tensor->name, ggml_op_name(tensor->op)); + } + for (const ggml_tensor * node : nodes) { + LLAMA_LOG_DEBUG("%s: input tensor '%32s' [%s, ne = { %5" PRId64 ", %5" PRId64 ", %5" PRId64 ", %5" PRId64 " }] is used by node '%s' (%s)\n", + __func__, tensor->name, ggml_type_name(tensor->type), + tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], + node->name, ggml_op_name(node->op)); + } + } + + return (int) users.size(); +} + void llama_context::sched_reserve() { if (!sched_need_reserve) { return; @@ -1804,11 +1839,15 @@ void llama_context::sched_reserve() { resolve_fused_ops(mctx.get(), n_seqs); // reserve worst-case graph - int n_splits_pp = -1; - int n_nodes_pp = -1; + int n_splits_pp = -1; + int n_nodes_pp = -1; + int n_inputs_pp = -1; + int n_input_tensors_pp = -1; - int n_splits_tg = -1; - int n_nodes_tg = -1; + int n_splits_tg = -1; + int n_nodes_tg = -1; + int n_inputs_tg = -1; + int n_input_tensors_tg = -1; // DSpark/MTP token-path graphs include a full-vocab lm_head. Prefill uses // the embd-inject path (logits off). Reserving n_outputs == n_ubatch here @@ -1837,8 +1876,10 @@ void llama_context::sched_reserve() { } } - n_splits_pp = ggml_backend_sched_get_n_splits(sched.get()); - n_nodes_pp = ggml_graph_n_nodes(gf); + n_splits_pp = ggml_backend_sched_get_n_splits(sched.get()); + n_nodes_pp = ggml_graph_n_nodes(gf); + n_inputs_pp = get_gf_res_reserve()->inputs.size(); + n_input_tensors_pp = this->n_input_tensors; } // reserve with tg (token generation) graph to get the number of splits and nodes @@ -1848,8 +1889,10 @@ void llama_context::sched_reserve() { throw std::runtime_error("failed to allocate compute tg buffers"); } - n_splits_tg = ggml_backend_sched_get_n_splits(sched.get()); - n_nodes_tg = ggml_graph_n_nodes(gf); + n_splits_tg = ggml_backend_sched_get_n_splits(sched.get()); + n_nodes_tg = ggml_graph_n_nodes(gf); + n_inputs_tg = get_gf_res_reserve()->inputs.size(); + n_input_tensors_tg = this->n_input_tensors; } // reserve again with pp graph to avoid ggml-alloc reallocations during inference @@ -1858,7 +1901,9 @@ void llama_context::sched_reserve() { // need to implement a more robust mechanism that tries a few different inputs and analyzes the results ggml_cgraph * gf = nullptr; switch (model.arch) { + case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_MINIMAX_01: + // [TAG_RESERVE_DIAG_DECAY] // the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which // makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1` gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc); @@ -1885,16 +1930,21 @@ void llama_context::sched_reserve() { } } - if (n_nodes_pp == n_nodes_tg) { - LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, n_nodes_pp); - } else { - LLAMA_LOG_INFO("%s: graph nodes = %d (with bs=%d), %d (with bs=1)\n", __func__, n_nodes_pp, n_tokens, n_nodes_tg); - } + { + const bool diff = n_nodes_pp != n_nodes_tg || n_splits_pp != n_splits_tg || + n_inputs_pp != n_inputs_tg || n_input_tensors_pp != n_input_tensors_tg; - if (n_splits_pp == n_splits_tg) { - LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits_pp); - } else { - LLAMA_LOG_INFO("%s: graph splits = %d (with bs=%d), %d (with bs=1)\n", __func__, n_splits_pp, n_tokens, n_splits_tg); + const auto val = [diff](int v_pp, int v_tg) -> std::string { + return diff ? format("%d / %d", v_pp, v_tg) : format("%d", v_pp); + }; + + LLAMA_LOG_INFO("%s: graph%s: nodes = %s, splits = %s, input objects = %s, input tensors = %s\n", + __func__, + diff ? format(" (pp bs=%d, tg bs=%d)", n_tokens, n_seqs).c_str() : "", + val(n_nodes_pp, n_nodes_tg).c_str(), + val(n_splits_pp, n_splits_tg).c_str(), + val(n_inputs_pp, n_inputs_tg).c_str(), + val(n_input_tensors_pp, n_input_tensors_tg).c_str()); } // WP_GRAPH_RESULT_SLOTS: see llama_context::wp_graph_slot in llama-context.h. @@ -5890,6 +5940,9 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { if (model.arch == LLM_ARCH_KIMI_K3) { // the n_tokens*40 budget below is exhausted at ubatch 3840 res = std::max<uint32_t>(n_tokens * 160, 64u * model.n_tensors()); + } else if (model.arch == LLM_ARCH_HRM_TEXT) { + // the 128-slot looped graph needs roughly one stack per token budget + res = std::max<uint32_t>(n_tokens * 80, 64u * model.n_tensors()); } else if (model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || model.arch == LLM_ARCH_BAILINGMOE3 || @@ -6045,6 +6098,7 @@ ggml_cgraph * llama_context::graph_reserve( auto * gf = model.build_graph(gparams); + this->n_input_tensors = llama_graph_n_input_tensors(gf); this->n_outputs = save_n_outputs; // initialize scheduler with the specified graph @@ -7911,6 +7965,9 @@ llama_context * llama_init_from_model( LLAMA_LOG_ERROR("%s: SPLIT_MODE_TENSOR requires flash_attn to be enabled\n", __func__); return nullptr; } + if (model->get_split_state_ud.n_devices == 1) { + LLAMA_LOG_WARN("%s: SPLIT_MODE_TENSOR being used for a single device is not recommended\n", __func__); + } } if ((model->hparams.is_mla() || model->arch == LLM_ARCH_DEEPSEEK4) && params.type_k != params.type_v) { diff --git a/src/llama-context.h b/src/llama-context.h index 74017fba45ce..8a9d0b9c5091 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -11,6 +11,7 @@ #include "ggml-cpp.h" #include "ggml-opt.h" +#include <array> #include <map> #include <vector> @@ -438,6 +439,7 @@ struct llama_context { uint64_t wp_last_graph_compute_ns() const { return wp_last_gc_ns; } private: + llm_graph_params graph_params( llm_graph_result * res, const llama_ubatch & ubatch, @@ -686,6 +688,7 @@ struct llama_context { // reuse the batch_allocr to avoid unnecessary memory allocations std::unique_ptr<llama_batch_allocr> balloc; + uint32_t n_input_tensors = 0; // number of tensors marked as input during the last graph reserve uint32_t n_outputs = 0; // number of actually-used outputs in the current ubatch or last logical batch std::vector<int32_t> output_ids; // map batch token positions to ids of the logits and embd buffers diff --git a/src/llama-grammar.cpp b/src/llama-grammar.cpp index f14215ac7e34..deffec8c0741 100644 --- a/src/llama-grammar.cpp +++ b/src/llama-grammar.cpp @@ -492,7 +492,7 @@ const char * llama_grammar_parser::parse_sequence( total_rules = min_times; } - if (n_prev_rules * total_rules >= MAX_REPETITION_THRESHOLD) { + if (n_prev_rules * total_rules > MAX_REPETITION_THRESHOLD) { throw std::runtime_error("number of rules that are going to be repeated multiplied by the new repetition exceeds sane defaults, please reduce the number of repetitions or rule complexity"); } @@ -871,17 +871,18 @@ static void llama_grammar_advance_stack( std::set<llama_grammar_stack, decltype(stack_cmp)> seen(stack_cmp); while (!todo.empty()) { - llama_grammar_stack curr_stack = std::move(todo.back()); + llama_grammar_stack curr_stack_candidate = std::move(todo.back()); todo.pop_back(); - if (seen.find( curr_stack) != seen.end()) { + auto [curr_stack_it, inserted] = seen.insert(std::move(curr_stack_candidate)); + if (!inserted) { continue; } - seen.insert(curr_stack); + const llama_grammar_stack & curr_stack = *curr_stack_it; if (curr_stack.empty()) { if (std::find(new_stacks.begin(), new_stacks.end(), curr_stack) == new_stacks.end()) { - new_stacks.emplace_back(std::move(curr_stack)); + new_stacks.emplace_back(curr_stack); } continue; } @@ -924,7 +925,7 @@ static void llama_grammar_advance_stack( case LLAMA_GRETYPE_TOKEN_NOT: if (std::find(new_stacks.begin(), new_stacks.end(), curr_stack) == new_stacks.end()) { // only add the stack if it's not a duplicate of one we already have - new_stacks.emplace_back(std::move(curr_stack)); + new_stacks.emplace_back(curr_stack); } break; default: diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index c0cd6def93e7..b042b1567ca2 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2074,8 +2074,26 @@ llm_graph_qkv llm_graph_context::build_qkv( int64_t n_head, int64_t n_head_kv, int il) const { - const int64_t n_embd_q = n_embd_head * n_head; - const int64_t n_embd_kv = n_embd_head * n_head_kv; + return build_qkv(layer, cur, + n_embd_head, n_head, + n_embd_head, n_head_kv, + n_embd_head, n_head_kv, + il); +} + +llm_graph_qkv llm_graph_context::build_qkv( + const llama_layer & layer, + ggml_tensor * cur, + int64_t n_embd_head_q, + int64_t n_head_q, + int64_t n_embd_head_k, + int64_t n_head_k, + int64_t n_embd_head_v, + int64_t n_head_v, + int il, + bool reshape) const { + const int64_t n_embd_q = n_embd_head_q * n_head_q; + const int64_t n_embd_k = n_embd_head_k * n_head_k; ggml_tensor * Qcur, * Kcur, * Vcur; @@ -2086,59 +2104,93 @@ llm_graph_qkv llm_graph_context::build_qkv( if (layer.wqkv_b) { qkv = ggml_add(ctx0, qkv, layer.wqkv_b); cb(qkv, "wqkv_b", il); + } else if (layer.wq_b && layer.wk_b && layer.wv_b) { + // Fused weights may coexist with separate Q/K/V biases in legacy or custom GGUFs. + ggml_tensor * qkv_b = ggml_concat(ctx0, ggml_concat(ctx0, layer.wq_b, layer.wk_b, 0), layer.wv_b, 0); + qkv = ggml_add(ctx0, qkv, qkv_b); + cb(qkv, "wqkv_b", il); } - if (hparams.f_clamp_kqv > 0.0f) { + if (reshape && hparams.f_clamp_kqv > 0.0f) { qkv = ggml_clamp(ctx0, qkv, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); cb(qkv, "wqkv_clamped", il); } - Qcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head, n_tokens, - ggml_row_size(qkv->type, n_embd_head), qkv->nb[1], 0); - Kcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head_kv, n_tokens, - ggml_row_size(qkv->type, n_embd_head), qkv->nb[1], - ggml_row_size(qkv->type, n_embd_q)); - Vcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head_kv, n_tokens, - ggml_row_size(qkv->type, n_embd_head), qkv->nb[1], - ggml_row_size(qkv->type, n_embd_q + n_embd_kv)); + if (reshape) { + Qcur = ggml_view_3d(ctx0, qkv, n_embd_head_q, n_head_q, n_tokens, + ggml_row_size(qkv->type, n_embd_head_q), qkv->nb[1], 0); + Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_k, n_tokens, + ggml_row_size(qkv->type, n_embd_head_k), qkv->nb[1], + ggml_row_size(qkv->type, n_embd_q)); + Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_v, n_tokens, + ggml_row_size(qkv->type, n_embd_head_v), qkv->nb[1], + ggml_row_size(qkv->type, n_embd_q + n_embd_k)); + } else { + Qcur = ggml_view_2d(ctx0, qkv, n_embd_q, n_tokens, qkv->nb[1], 0); + Kcur = ggml_view_2d(ctx0, qkv, n_embd_k, n_tokens, qkv->nb[1], + ggml_row_size(qkv->type, n_embd_q)); + Vcur = ggml_view_2d(ctx0, qkv, n_embd_head_v * n_head_v, n_tokens, qkv->nb[1], + ggml_row_size(qkv->type, n_embd_q + n_embd_k)); + } + if (!reshape) { + Qcur = ggml_cont(ctx0, Qcur); + Kcur = ggml_cont(ctx0, Kcur); + Vcur = ggml_cont(ctx0, Vcur); + } } else { // separate Q/K/V path Qcur = build_lora_mm(layer.wq, cur, layer.wq_s); - cb(Qcur, "Qcur", il); + if (reshape) { + cb(Qcur, "Qcur", il); + } if (layer.wq_b) { Qcur = ggml_add(ctx0, Qcur, layer.wq_b); - cb(Qcur, "Qcur", il); + if (reshape) { + cb(Qcur, "Qcur", il); + } } - if (hparams.f_clamp_kqv > 0.0f) { + if (reshape && hparams.f_clamp_kqv > 0.0f) { Qcur = ggml_clamp(ctx0, Qcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); cb(Qcur, "Qcur_clamped", il); } Kcur = build_lora_mm(layer.wk, cur, layer.wk_s); - cb(Kcur, "Kcur", il); + if (reshape) { + cb(Kcur, "Kcur", il); + } if (layer.wk_b) { Kcur = ggml_add(ctx0, Kcur, layer.wk_b); - cb(Kcur, "Kcur", il); + if (reshape) { + cb(Kcur, "Kcur", il); + } } - if (hparams.f_clamp_kqv > 0.0f) { + if (reshape && hparams.f_clamp_kqv > 0.0f) { Kcur = ggml_clamp(ctx0, Kcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); cb(Kcur, "Kcur_clamped", il); } Vcur = build_lora_mm(layer.wv, cur, layer.wv_s); - cb(Vcur, "Vcur", il); + if (reshape) { + cb(Vcur, "Vcur", il); + } if (layer.wv_b) { Vcur = ggml_add(ctx0, Vcur, layer.wv_b); - cb(Vcur, "Vcur", il); + if (reshape) { + cb(Vcur, "Vcur", il); + } } - if (hparams.f_clamp_kqv > 0.0f) { + if (reshape && hparams.f_clamp_kqv > 0.0f) { Vcur = ggml_clamp(ctx0, Vcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); cb(Vcur, "Vcur_clamped", il); } - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + if (reshape) { + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_q, n_head_q, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_k, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head_v, n_head_v, n_tokens); + } } - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); + if (reshape) { + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + } return { Qcur, Kcur, Vcur }; } @@ -2325,7 +2377,7 @@ ggml_tensor * llm_graph_context::build_ffn( cur = build_lora_mm(down, cur); if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) { // GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators - ggml_mul_mat_set_prec(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); } } @@ -2423,7 +2475,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( if (probs_in == nullptr) { logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens] if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) { - ggml_mul_mat_set_prec(logits, GGML_PREC_F32); + ggml_prec_set_acc(logits, GGML_PREC_F32); } cb(logits, "ffn_moe_logits", il); } else { @@ -2687,7 +2739,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( const float limit = hparams.swiglu_clamp_exp[il]; constexpr float eps = 1e-6f; if (limit > eps) { - if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) { + if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) { cur = ggml_swiglu_clamp(ctx0, cur, up, limit); } else { up = ggml_clamp(ctx0, up, -limit, limit); @@ -2764,6 +2816,10 @@ ggml_tensor * llm_graph_context::build_moe_ffn( } experts = build_lora_mm_id(down_exps, cur, selected_experts, down_exps_s); // [n_embd, n_expert_used, n_tokens] + if (arch == LLM_ARCH_MISTRAL4) { + // src1 can exceed F16 range + ggml_prec_set_src(experts, GGML_PREC_F32, 1); + } cb(experts, "ffn_moe_down", il); if (down_exps_s) { @@ -3105,6 +3161,7 @@ ggml_tensor * llm_graph_context::build_inp_pos() const { cur = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, (int64_t)n_tokens*hparams.n_pos_per_embd()); ggml_set_input(cur); + cb(cur, "inp_pos", -1); res->add_input(std::move(inp)); @@ -3119,7 +3176,7 @@ ggml_tensor * llm_graph_context::build_inp_attn_scale() const { // this need to be 1x1xN for broadcasting cur = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, 1, n_tokens); ggml_set_input(cur); - ggml_set_name(cur, "attn_scale"); + cb(cur, "inp_attn_scale", -1); res->add_input(std::move(inp)); @@ -3141,6 +3198,7 @@ ggml_tensor * llm_graph_context::build_inp_out_ids(int64_t row_offset) const { cur = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_outputs); ggml_set_input(cur); + ggml_set_name(cur, "out_ids"); res->add_input(std::move(inp)); @@ -3187,6 +3245,7 @@ ggml_tensor * llm_graph_context::build_inp_mean() const { cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, ubatch.n_seqs_unq); ggml_set_input(cur); + ggml_set_name(cur, "mean"); res->add_input(std::move(inp)); @@ -3200,6 +3259,7 @@ ggml_tensor * llm_graph_context::build_inp_cls() const { cur = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_seqs_unq); ggml_set_input(cur); + ggml_set_name(cur, "cls"); res->add_input(std::move(inp)); @@ -3224,6 +3284,7 @@ ggml_tensor * llm_graph_context::build_inp_cross_embd() const { cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_enc); ggml_set_input(cur); + ggml_set_name(cur, "cross_embd"); res->add_input(std::move(inp)); @@ -3237,6 +3298,7 @@ ggml_tensor * llm_graph_context::build_inp_pos_bucket_enc() const { cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_tokens, n_tokens); ggml_set_input(cur); + ggml_set_name(cur, "pos_bucket_enc"); res->add_input(std::move(inp)); @@ -3254,6 +3316,7 @@ ggml_tensor * llm_graph_context::build_inp_pos_bucket_dec() const { cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_kv, n_tokens); ggml_set_input(cur); + ggml_set_name(cur, "pos_bucket_dec"); res->add_input(std::move(inp)); @@ -3327,7 +3390,7 @@ ggml_tensor * llm_graph_context::build_attn_mha( ggml_flash_attn_ext_add_sinks(cur, sinks); GGML_ASSERT(n_kv_max >= 0 && n_kv_max <= INT32_MAX); ggml_flash_attn_ext_set_n_kv_max(cur, static_cast<int32_t>(n_kv_max)); - ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); // TurboQuant: inverse WHT on FA output when V values are WHT-rotated. // For MLA, V is a view of K with different ne[0] (e.g. V=512, K=576). @@ -3367,7 +3430,7 @@ ggml_tensor * llm_graph_context::build_attn_mha( // note: this op tends to require high floating point range // while for some models F16 is enough, for others it is not, so we default to F32 here - ggml_mul_mat_set_prec(kq, GGML_PREC_F32); + ggml_prec_set_acc(kq, GGML_PREC_F32); if (arch == LLM_ARCH_GROK) { // need to do the following: @@ -3454,6 +3517,7 @@ llm_graph_input_attn_no_cache * llm_graph_context::build_attn_inp_no_cache() con // note: there is no KV cache, so the number of KV values is equal to the number of tokens in the batch inp->self_kq_mask = ggml_new_tensor_4d(ctx0, type_mask, n_tokens, n_tokens, 1, 1); ggml_set_input(inp->self_kq_mask); + cb(inp->self_kq_mask, "self_kq_mask", -1); inp->self_kq_mask_cnv = inp->self_kq_mask; @@ -3825,7 +3889,7 @@ ggml_tensor * llm_graph_context::build_attn( if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) { // GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators cur = build_lora_mm(wo, cur); - ggml_mul_mat_set_prec(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); if (wo_s) { cur = ggml_mul(ctx0, cur, wo_s); } @@ -3945,7 +4009,7 @@ ggml_tensor * llm_graph_context::build_attn( if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) { // GLM4 and GLM4_MOE seem to have numerical issues with half-precision accumulators cur = build_lora_mm(wo, cur); - ggml_mul_mat_set_prec(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); if (wo_s) { cur = ggml_mul(ctx0, cur, wo_s); } @@ -4503,6 +4567,7 @@ static std::unique_ptr<llm_graph_input_rs> build_rs_inp_impl( inp->s_copy = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_rs); ggml_set_input(inp->s_copy); + ggml_set_name(inp->s_copy, "rs_s_copy"); inp->s_copy_main = ggml_view_1d(ctx0, inp->s_copy, n_seqs, 0); inp->s_copy_extra = ggml_view_1d(ctx0, inp->s_copy, n_rs - n_seqs, n_seqs * inp->s_copy->nb[0]); diff --git a/src/llama-graph.h b/src/llama-graph.h index f233983976cc..ac416ea9ed44 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -7,6 +7,7 @@ #include "llama-ml8-registry.h" #include <cstdint> +#include <cstdlib> #include <vector> #include <memory> #include <set> @@ -1458,6 +1459,19 @@ struct llm_graph_context { int64_t n_head_kv, int il) const; + // Set reshape to false to return contiguous projections before clamp/reshape. + llm_graph_qkv build_qkv( + const llama_layer & layer, + ggml_tensor * cur, + int64_t n_embd_head_q, + int64_t n_head_q, + int64_t n_embd_head_k, + int64_t n_head_k, + int64_t n_embd_head_v, + int64_t n_head_v, + int il, + bool reshape = true) const; + ggml_tensor * build_ffn( ggml_tensor * cur, ggml_tensor * up, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 15092b04f819..280672f875fc 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -209,6 +209,12 @@ struct llama_hparams { float situ_beta = 1.0f; float situ_linear_beta = 0.0f; // 0 = no linear-beta transform on the up branch + // hrm-text (looped H/L stacks) + uint32_t n_hrm_layers_per_stack = 0; + uint32_t n_hrm_h_cycles = 0; + uint32_t n_hrm_l_cycles = 0; + bool hrm_prefix_lm = false; + bool ssm_dt_b_c_rms = false; float f_clamp_kqv = 0.0f; diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 2ad0ea359296..605e8122b3b5 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1880,6 +1880,7 @@ ggml_tensor * llama_kv_cache::build_input_k_idxs(ggml_context * ctx, const llama ggml_tensor * k_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens); ggml_set_input(k_idxs); + ggml_set_name(k_idxs, "attn_inp_k_idxs"); return k_idxs; } @@ -1896,6 +1897,7 @@ ggml_tensor * llama_kv_cache::build_input_v_idxs(ggml_context * ctx, const llama } ggml_set_input(v_idxs); + ggml_set_name(v_idxs, "attn_inp_v_idxs"); return v_idxs; } diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cpp index 1bd9fd3ccbce..f2ec474bef71 100644 --- a/src/llama-memory-hybrid-idx.cpp +++ b/src/llama-memory-hybrid-idx.cpp @@ -64,6 +64,10 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx( // K-shift must not rotate them while the stream copies in the same update still apply hparams_idx.rope_type = LLAMA_ROPE_TYPE_NONE; + // fool llama_kv_cache into thinking this is a MLA cache, so it won't cache V tensors + hparams_idx.n_embd_head_k_mla_impl = model.hparams.indexer_head_size; + hparams_idx.n_embd_head_v_mla_impl = model.hparams.indexer_head_size; + LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size); return new llama_kv_cache( diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index 4d183cbc9c45..715a6e3548e6 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -6,6 +6,7 @@ #include <cstring> #include <climits> +#include <cstdlib> #include <stdexcept> #include <cerrno> #include <algorithm> diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index ad93e94e1565..4e942d2fbd73 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -725,6 +725,13 @@ llama_model_loader::llama_model_loader( throw std::runtime_error(format("%s: failed to load model from file pointer", __func__)); } + // mmap places tensors at their file offsets, so an embedded GGUF must be aligned in the file too + const size_t tensor_align = ggml_backend_buft_get_alignment(ggml_backend_cpu_buffer_type()); + if (use_mmap && gguf_get_data_offset(metadata) % tensor_align != 0) { + throw std::runtime_error(format("%s: GGUF data section at file offset %zu is not %zu byte aligned, cannot mmap", + __func__, gguf_get_data_offset(metadata), tensor_align)); + } + get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false); llm_kv = LLM_KV(llm_arch_from_string(arch_name)); diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index e64d4cc8c817..d85080cf03f5 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -11,27 +11,15 @@ #include <cstdint> #include <string> +#include <unordered_set> bool llama_model_saver_supports_arch(llm_arch arch) { switch (arch) { - case LLM_ARCH_PLAMO3: - case LLM_ARCH_GEMMA3: case LLM_ARCH_GEMMA3N: - case LLM_ARCH_COHERE2: - case LLM_ARCH_COHERE2MOE: - case LLM_ARCH_OLMO2: case LLM_ARCH_BITNET: case LLM_ARCH_T5: - case LLM_ARCH_EXAONE_MOE: - case LLM_ARCH_AFMOE: case LLM_ARCH_APERTUS: - case LLM_ARCH_MIMO2: case LLM_ARCH_STEP35: - case LLM_ARCH_MUSE_GLIMMER: - case LLM_ARCH_MELLUM: - case LLM_ARCH_LAGUNA: - case LLM_ARCH_GRANITE_SWA: - case LLM_ARCH_DOTS3NOTE: // TODO: need to handle SWA pattern and MLA+SWA config return false; default: return true; @@ -259,6 +247,10 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_TIME_DECAY_EXTRA_DIM, hparams.time_decay_extra_dim); add_kv(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale); add_kv(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale); + add_kv(LLM_KV_HRM_LAYERS_PER_STACK, hparams.n_hrm_layers_per_stack); + add_kv(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles); + add_kv(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles); + add_kv(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm); add_kv(LLM_KV_TOKEN_SHIFT_COUNT, hparams.token_shift_count); add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step); // add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, ???); // saved as LLM_KV_ATTENTION_RECURRENT_LAYERS instead @@ -283,7 +275,11 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts); add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true); add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); - // add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???); + if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { + // never collapsed to a scalar: the loaders read a scalar as a period + add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>( + hparams.is_swa_impl.begin(), hparams.is_swa_impl.begin() + hparams.n_layer_all)); + } add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale); add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale); add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale); @@ -293,6 +289,9 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl); add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa); add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa); + add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, hparams.n_embd_head_k_mla_swa); + add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, hparams.n_embd_head_v_mla_swa); + add_kv(LLM_KV_ATTENTION_KV_LORA_RANK_SWA, hparams.n_lora_kv_swa); add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k); @@ -418,13 +417,13 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_TOKENIZER_SCORES, scores); add_kv(LLM_KV_TOKENIZER_MERGES, vocab.get_bpe_merges()); // FIXME llama_token is type i32 but when reading in a GGUF file u32 is expected, not an issue for writing though - add_kv(LLM_KV_TOKENIZER_BOS_ID, uint32_t(vocab.token_bos())); - add_kv(LLM_KV_TOKENIZER_EOS_ID, uint32_t(vocab.token_eos())); - add_kv(LLM_KV_TOKENIZER_EOT_ID, uint32_t(vocab.token_eot())); - add_kv(LLM_KV_TOKENIZER_EOM_ID, uint32_t(vocab.token_eom())); - add_kv(LLM_KV_TOKENIZER_UNK_ID, uint32_t(vocab.token_unk())); - add_kv(LLM_KV_TOKENIZER_SEP_ID, uint32_t(vocab.token_sep())); - add_kv(LLM_KV_TOKENIZER_PAD_ID, uint32_t(vocab.token_pad())); + if (vocab.token_bos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_BOS_ID, uint32_t(vocab.token_bos())); } + if (vocab.token_eos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOS_ID, uint32_t(vocab.token_eos())); } + if (vocab.token_eot() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOT_ID, uint32_t(vocab.token_eot())); } + if (vocab.token_eom() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOM_ID, uint32_t(vocab.token_eom())); } + if (vocab.token_unk() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_UNK_ID, uint32_t(vocab.token_unk())); } + if (vocab.token_sep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_SEP_ID, uint32_t(vocab.token_sep())); } + if (vocab.token_pad() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_PAD_ID, uint32_t(vocab.token_pad())); } // add_kv(LLM_KV_TOKENIZER_CLS_ID, uint32_t(vocab.token_bos())); // deprecated // add_kv(LLM_KV_TOKENIZER_MASK_ID, ???); add_kv(LLM_KV_TOKENIZER_ADD_BOS, vocab.get_add_bos()); @@ -435,12 +434,12 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap()); // add_kv(LLM_KV_TOKENIZER_HF_JSON, ???); // add_kv(LLM_KV_TOKENIZER_RWKV, ???); - add_kv(LLM_KV_TOKENIZER_FIM_PRE_ID, uint32_t(vocab.token_fim_pre())); - add_kv(LLM_KV_TOKENIZER_FIM_SUF_ID, uint32_t(vocab.token_fim_suf())); - add_kv(LLM_KV_TOKENIZER_FIM_MID_ID, uint32_t(vocab.token_fim_mid())); - add_kv(LLM_KV_TOKENIZER_FIM_PAD_ID, uint32_t(vocab.token_fim_pad())); - add_kv(LLM_KV_TOKENIZER_FIM_REP_ID, uint32_t(vocab.token_fim_rep())); - add_kv(LLM_KV_TOKENIZER_FIM_SEP_ID, uint32_t(vocab.token_fim_sep())); + if (vocab.token_fim_pre() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_PRE_ID, uint32_t(vocab.token_fim_pre())); } + if (vocab.token_fim_suf() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_SUF_ID, uint32_t(vocab.token_fim_suf())); } + if (vocab.token_fim_mid() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_MID_ID, uint32_t(vocab.token_fim_mid())); } + if (vocab.token_fim_pad() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_PAD_ID, uint32_t(vocab.token_fim_pad())); } + if (vocab.token_fim_rep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_REP_ID, uint32_t(vocab.token_fim_rep())); } + if (vocab.token_fim_sep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_SEP_ID, uint32_t(vocab.token_fim_sep())); } // TODO: implement LoRA support // add_kv(LLM_KV_ADAPTER_TYPE, ???); @@ -503,6 +502,7 @@ void llama_model_saver::add_tensors_from_model() { add_tensor(model->cls_out); add_tensor(model->cls_out_b); add_tensor(model->cls_norm); + add_tensor(model->hrm_z_l_init); add_tensor(model->hc_head_fn); add_tensor(model->hc_head_base); add_tensor(model->hc_head_scale); @@ -511,9 +511,17 @@ void llama_model_saver::add_tensors_from_model() { add_tensor(model->hc_head_down); add_tensor(model->hc_head_up); + // looped architectures alias physical tensors across cache slots; save each + // tensor once. a different tensor with an existing name still asserts below + std::unordered_set<const struct ggml_tensor *> seen; + for (const struct llama_layer & layer : model->layers) { for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) { - add_tensor(reinterpret_cast<const struct ggml_tensor * const *>(&layer)[i]); + const struct ggml_tensor * tensor = reinterpret_cast<const struct ggml_tensor * const *>(&layer)[i]; + if (tensor == nullptr || !seen.insert(tensor).second) { + continue; + } + add_tensor(tensor); } } } diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 796d2341d666..64f2812da76f 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -198,6 +198,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_mamba(params); case LLM_ARCH_MAMBA2: return new llama_model_mamba2(params); + case LLM_ARCH_MAPLE: + return new llama_model_maple(params); case LLM_ARCH_JAMBA: return new llama_model_jamba(params); case LLM_ARCH_XVERSE: @@ -350,6 +352,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_minimax_m2(params); case LLM_ARCH_MINIMAX_M3: return new llama_model_minimax_m3(params); + case LLM_ARCH_HRM_TEXT: + return new llama_model_hrm_text(params); case LLM_ARCH_COGVLM: return new llama_model_cogvlm(params); case LLM_ARCH_PANGU_EMBED: @@ -380,6 +384,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_kimi_k3(params); case LLM_ARCH_STEP35: return new llama_model_step35(params); + case LLM_ARCH_SPARK2_5: + return new llama_model_spark2_5(params); default: throw std::runtime_error(std::string("unsupported model architecture: '") + llm_arch_name(arch) + "'"); } @@ -565,6 +571,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str }; auto get_tensor_config = [&]() -> tensor_config { + if (ud->model->arch == LLM_ARCH_HRM_TEXT) { + // aliased cache slots cannot satisfy the meta-split invariants, so replicate all tensors + return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, tensor, 0, 0}; + } // NextN / MTP block. // // The MTP block's transformer tensors are named exactly like a trunk dense @@ -771,8 +781,20 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str }; auto get_split_segments = [&](int axis, uint32_t il) -> std::vector<std::pair<int64_t, uint32_t>> { + // TODO: clarify why this is necessary specifically for these models + // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN] if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_QWEN4EXP) { + + // fused full attention layers with Q gate tensors that need n_embd doubled: + if (!hparams.is_recr(il) && (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias))) { + const int64_t n_embd = hparams.n_head(il) * hparams.n_embd_head_k(il) * 2; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(il); + GGML_ASSERT(hparams.n_embd_k_gqa(il) == n_embd_gqa); + GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa); + return {{n_embd, 1}, {n_embd_gqa, 2}}; + } + const int64_t head_k_dim = hparams.ssm_d_state; const int64_t head_v_dim = hparams.ssm_d_state; const int64_t n_k_heads = hparams.ssm_n_group; @@ -822,9 +844,9 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str } if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) { - const int64_t n_embd = hparams.n_embd; + const int64_t n_embd = hparams.n_head(il) * hparams.n_embd_head_k(il); const int64_t n_embd_gqa = hparams.n_embd_v_gqa(il); - GGML_ASSERT(hparams.n_embd_k_gqa() == n_embd_gqa); + GGML_ASSERT(hparams.n_embd_k_gqa(il) == n_embd_gqa); GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa); return {{n_embd, 1}, {n_embd_gqa, 2}}; } @@ -910,6 +932,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_q_bias)) { GGML_ASSERT(segments.size() == 1); // some models have Q gate tensors, for those cases the granularity needs to be doubled: + // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN] if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_QWEN4EXP) { return {std::lcm(2*n_embd_q, blck_size_perf)}; @@ -949,6 +972,12 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str } if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) { GGML_ASSERT(segments.size() == 2); + // fused full attention layers need Q gate tensors handled like above: + // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN] + if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || + ud->model->arch == LLM_ARCH_QWEN4EXP) { + return {std::lcm(2*n_embd_q, blck_size_perf), granularity_kv}; + } return {granularity_q, granularity_kv}; } } @@ -1230,6 +1259,8 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_17B_16E: return "17Bx16E (Scout)"; case LLM_TYPE_17B_128E: return "17Bx128E (Maverick)"; case LLM_TYPE_A13B: return "A13B"; + case LLM_TYPE_1B_A400M: return "1B.A400M"; + case LLM_TYPE_3B_A800M: return "3B.A800M"; case LLM_TYPE_7B_A1B: return "7B.A1B"; case LLM_TYPE_8B_A1B: return "8B.A1B"; case LLM_TYPE_7_9B_A1_3B: return "7.9B.A1.3B"; @@ -1240,6 +1271,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_26B_A4B: return "26B.A4B"; case LLM_TYPE_30B_A3B: return "30B.A3B"; case LLM_TYPE_31B_A3_5B: return "31B.A3.5B"; + case LLM_TYPE_32B_A9B: return "32B.A9B"; case LLM_TYPE_35B_A3B: return "35B.A3B"; case LLM_TYPE_48B_A3B: return "48B.A3B"; case LLM_TYPE_75B_A9B: return "75B.A9B"; @@ -2362,6 +2394,18 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } } + // resolve AUTO on systems without mmap support (e.g. iGPUs): fall back to OFF; see #28160 + if (ml.lazy.mode == LLAMA_LAZY_MODE_AUTO) { + for (const auto & dev : devices) { + ggml_backend_dev_props props; + ggml_backend_dev_get_props(dev.dev, &props); + if (!props.caps.mmap_support) { + ml.lazy.mode = LLAMA_LAZY_MODE_OFF; + break; + } + } + } + const char * load_mode_name = params.load_mode == LLAMA_LOAD_MODE_AUTO ? llama_load_mode_name(ml.use_mmap ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE) : llama_load_mode_name(params.load_mode); @@ -4324,9 +4368,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, filter_authoritative = true; } - if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA || - arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_DEEPSEEK32) && - hparams.n_layer_nextn > 0) { + // don't filter when n_layer_nextn is repurposed for a router layer the trunk attends + // or when a model is entirely n_layer_nextn layers and has no trunk + if (hparams.n_layer_nextn > 0 && hparams.n_layer() > 0 && hparams.router_layer < 0) { if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; filter_authoritative = true; @@ -5025,8 +5069,11 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_QWEN3NEXT: case LLM_ARCH_MIMO2: case LLM_ARCH_STEP35: + case LLM_ARCH_SPARK2_5: case LLM_ARCH_TALKIE: case LLM_ARCH_MELLUM: + case LLM_ARCH_MAPLE: + case LLM_ARCH_HRM_TEXT: return LLAMA_ROPE_TYPE_NEOX; case LLM_ARCH_DFLASH: @@ -5252,6 +5299,15 @@ ggml_tensor * llama_model_base::create_tensor(const LLM_TN_IMPL & tn, const std: } void llama_model_base::create_tensor_gate_up_exps(llama_layer & layer, int bid, int64_t n_embd_, int64_t n_ff_, int64_t n_expert_, int flags) { + if (flags & TENSOR_SKIP) { + const int skip = TENSOR_NOT_REQUIRED | TENSOR_SKIP; + + create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", bid), {n_embd_, n_ff_ * 2, n_expert_}, skip | TENSOR_SKIP_IF_VIRTUAL); + create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", bid), {n_embd_, n_ff_, n_expert_}, skip); + create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", bid), {n_embd_, n_ff_, n_expert_}, skip); + return; + } + layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", bid), {n_embd_, n_ff_ * 2, n_expert_}, TENSOR_NOT_REQUIRED); if (layer.ffn_gate_up_exps == nullptr) { layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", bid), {n_embd_, n_ff_, n_expert_}, flags); @@ -5281,6 +5337,12 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid, layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); if (layer.wqkv) { layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); + // Fused weights may coexist with separate Q/K/V biases in legacy or custom GGUFs. + if (!layer.wqkv_b) { + layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", bid), {n_embd_q_}, TENSOR_NOT_REQUIRED); + layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", bid), {n_embd_k_}, TENSOR_NOT_REQUIRED); + layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", bid), {n_embd_v_}, TENSOR_NOT_REQUIRED); + } } else { layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", bid), {n_embd_, n_embd_q_}, flags); layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", bid), {n_embd_, n_embd_k_}, flags); @@ -5291,6 +5353,15 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid, } } +void llama_model_base::load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first) { + if (ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, false)) { + return; + } + + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern, false); + hparams.set_swa_pattern(n_pattern, dense_first); +} + const int32_t * llama_model_target_layer_ids(const struct llama_model * model) { const auto & v = model->target_layer_ids; return v.empty() ? nullptr : v.data(); diff --git a/src/llama-model.h b/src/llama-model.h index 94c83f033a78..e440d6a76400 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -120,6 +120,8 @@ enum llm_type { LLM_TYPE_17B_16E, // llama4 Scout LLM_TYPE_17B_128E, // llama4 Maverick LLM_TYPE_A13B, + LLM_TYPE_1B_A400M, // Granite3 MoE + LLM_TYPE_3B_A800M, // Granite3 MoE LLM_TYPE_7B_A1B, LLM_TYPE_8B_A1B, // lfm2moe LLM_TYPE_7_9B_A1_3B, // Ling-3.0-tiny @@ -130,6 +132,7 @@ enum llm_type { LLM_TYPE_26B_A4B, // Gemma4 LLM_TYPE_30B_A3B, LLM_TYPE_31B_A3_5B, + LLM_TYPE_32B_A9B, // Granite4 Hybrid LLM_TYPE_35B_A3B, // Qwen3.5 LLM_TYPE_48B_A3B, // Kimi Linear LLM_TYPE_75B_A9B, // Nemotron 3 Puzzle @@ -741,6 +744,9 @@ struct llama_model { struct ggml_tensor * nextn_proj_pre = nullptr; struct ggml_tensor * nextn_proj_post = nullptr; + // hrm-text initial low-cycle state + struct ggml_tensor * hrm_z_l_init = nullptr; + // DeepSeek-V4 struct ggml_tensor * hc_head_fn = nullptr; struct ggml_tensor * hc_head_base = nullptr; @@ -961,6 +967,9 @@ struct llama_model_base : public llama_model { int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_, int flags); + // helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_pattern + void load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false); + void load_stats (llama_model_loader & ml) override; void load_hparams(llama_model_loader & ml) override; void load_vocab (llama_model_loader & ml) override; diff --git a/src/llama-sampler.cpp b/src/llama-sampler.cpp index 34a7988262ea..b797c50bef62 100644 --- a/src/llama-sampler.cpp +++ b/src/llama-sampler.cpp @@ -644,7 +644,7 @@ static bool llama_sampler_backend_support( return true; } - auto probe = llama_sampler_backend_probe_graph(smpl, 1024*1024, GGML_DEFAULT_GRAPH_SIZE, true); + auto probe = llama_sampler_backend_probe_graph(smpl, 128*1024, GGML_DEFAULT_GRAPH_SIZE, true); for (int i = 0; i < ggml_graph_n_nodes(probe.gf); i++) { struct ggml_tensor * op = ggml_graph_node(probe.gf, i); @@ -764,7 +764,7 @@ static bool llama_sampler_chain_backend_init( res = res && cur_prefix; } - auto probe = llama_sampler_backend_probe_graph(smpl, 1024*1024, GGML_DEFAULT_GRAPH_SIZE, false); + auto probe = llama_sampler_backend_probe_graph(smpl, 128*1024, GGML_DEFAULT_GRAPH_SIZE, false); chain->n_nodes = llama_sampler_backend_probe_n_nodes(probe); return res; @@ -1006,7 +1006,7 @@ struct llama_sampler * llama_sampler_chain_remove(struct llama_sampler * chain, return result; } -int llama_sampler_chain_n(const struct llama_sampler * chain) { +int32_t llama_sampler_chain_n(const struct llama_sampler * chain) { const auto * p = (const llama_sampler_chain *) chain->ctx; return p->samplers.size(); diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index c0c34cdd8cd9..e038637ce707 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -14,6 +14,7 @@ #include <cmath> #include <cstdarg> #include <cstring> +#include <cstdlib> #include <forward_list> #include <limits> #include <map> @@ -325,6 +326,14 @@ struct llm_tokenizer_bpe : llm_tokenizer { "[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+[\r\n]*|\\s*[\r\n]+|\\s+(?!\\S)|\\s+", }; break; + case LLAMA_VOCAB_PRE_TYPE_SPARK2_5: + regex_exprs = { + "\\p{N}{1,3}", + "[一-龥぀-ゟ゠-ヿ]+", + "[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+|[\r\n]|\\s+(?!\\S)|\\s+", + "\\p{N}", + }; + break; case LLAMA_VOCAB_PRE_TYPE_YOUTU: regex_exprs = { "[가-힣ㄱ-ㆎ]+|[！…“”‘’—：；，、-〿︰-﹏]+|[ㄅ-ㄯ]+|[一-龥぀-ゟ゠-ヿ]+", @@ -479,6 +488,12 @@ struct llm_tokenizer_bpe : llm_tokenizer { "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1}| ?[^\\s\\p{L}\\p{N}\\r\\n]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", }; break; + case LLAMA_VOCAB_PRE_TYPE_UFAKZEKA: + regex_exprs = { + // Qwen2 pattern without the English contraction group, so Turkish apostrophe suffixes stay attached + "[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; + break; case LLAMA_VOCAB_PRE_TYPE_GROK_2: regex_exprs = { // original regex from tokenizer.json @@ -2072,6 +2087,16 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { special_unk_id = LLAMA_TOKEN_NULL; special_sep_id = LLAMA_TOKEN_NULL; special_pad_id = LLAMA_TOKEN_NULL; + } else if (tokenizer_model == "test") { + type = LLAMA_VOCAB_TYPE_TEST; + + // default special tokens + special_bos_id = LLAMA_TOKEN_NULL; + special_eos_id = LLAMA_TOKEN_NULL; + special_unk_id = LLAMA_TOKEN_NULL; + special_sep_id = LLAMA_TOKEN_NULL; + special_pad_id = LLAMA_TOKEN_NULL; + special_mask_id = LLAMA_TOKEN_NULL; } else if (tokenizer_model == "plamo2") { type = LLAMA_VOCAB_TYPE_PLAMO2; @@ -2170,6 +2195,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "deepseek-v3") { pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM; clean_spaces = false; + } else if ( + tokenizer_pre == "spark2_5") { + pre_type = LLAMA_VOCAB_PRE_TYPE_SPARK2_5; + clean_spaces = false; } else if ( tokenizer_pre == "youtu") { pre_type = LLAMA_VOCAB_PRE_TYPE_YOUTU; @@ -2363,6 +2392,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "kimi-k2") { pre_type = LLAMA_VOCAB_PRE_TYPE_KIMI_K2; clean_spaces = false; + } else if ( + tokenizer_pre == "ufakzeka") { + pre_type = LLAMA_VOCAB_PRE_TYPE_UFAKZEKA; + clean_spaces = false; } else if ( tokenizer_pre == "grok-2") { pre_type = LLAMA_VOCAB_PRE_TYPE_GROK_2; @@ -3111,6 +3144,7 @@ std::string llama_vocab::impl::type_name() const{ case LLAMA_VOCAB_TYPE_UGM: return "UGM"; case LLAMA_VOCAB_TYPE_RWKV: return "RWKV"; case LLAMA_VOCAB_TYPE_PLAMO2: return "PLaMo2"; + case LLAMA_VOCAB_TYPE_TEST: return "TEST"; default: return "unknown"; } } @@ -3199,6 +3233,9 @@ void llama_vocab::impl::init_tokenizer(enum llama_vocab_type type) { case LLAMA_VOCAB_TYPE_PLAMO2: tokenizer = std::make_unique<llm_tokenizer_plamo2>(vocab); break; + case LLAMA_VOCAB_TYPE_TEST: + tokenizer = std::make_unique<llm_tokenizer>(); + break; default: GGML_ABORT("unsupported vocab type"); } @@ -3572,6 +3609,42 @@ std::vector<llama_token> llama_vocab::impl::tokenize( } } } break; + case LLAMA_VOCAB_TYPE_TEST: + { + const uint32_t n_vocab = vocab.n_tokens(); + constexpr size_t chunk_size = 5; + + // reserve output to avoid repeated reallocations + size_t n_tokens = 0; + for (const auto & fragment : fragment_buffer) { + if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { + n_tokens += (fragment.length + chunk_size - 1) / chunk_size; + } else { + ++n_tokens; + } + } + output.reserve(output.size() + n_tokens); + + for (const auto & fragment : fragment_buffer) { + if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { + const auto & text = fragment.raw_text; + const size_t begin = fragment.offset; + const size_t end = begin + fragment.length; + size_t pos = begin; + while (pos < end) { + const size_t n = std::min(chunk_size, end - pos); + uint64_t hash = 0; + for (size_t i = 0; i < n; ++i) { + hash = hash*31 + (uint8_t) text[pos + i]; + } + output.push_back((llama_token)(hash % n_vocab)); + pos += n; + } + } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) + output.push_back(fragment.token); + } + } + } break; case LLAMA_VOCAB_TYPE_NONE: GGML_ABORT("fatal error"); } @@ -3670,6 +3743,11 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t memcpy(buf, result.data(), result.size()); return (int)result.size(); } + case LLAMA_VOCAB_TYPE_TEST: { + // tokens -> text: simply stringify the token id in hex + std::string result = format("%x", token); + return _try_copy(result.data(), result.size()); + } case LLAMA_VOCAB_TYPE_PLAMO2: { // PLaMo-2 uses similar token handling as BPE/SPM if (vocab.is_byte(token)) { @@ -3940,6 +4018,9 @@ llama_token llama_vocab::byte_to_token(uint8_t ch) const { snprintf(hex_str, sizeof(hex_str), "<0x%02X>", ch); return pimpl->token_to_id.at(hex_str); } + case LLAMA_VOCAB_TYPE_TEST: + // TEST tokens have no byte-level mapping + return LLAMA_TOKEN_NULL; default: GGML_ABORT("fatal error"); } diff --git a/src/llama-vocab.h b/src/llama-vocab.h index e02ea78ffaea..3fb061f0ecf1 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -66,6 +66,8 @@ enum llama_vocab_pre_type { LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55, LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56, LLAMA_VOCAB_PRE_TYPE_HY_V4 = 57, + LLAMA_VOCAB_PRE_TYPE_SPARK2_5 = 58, + LLAMA_VOCAB_PRE_TYPE_UFAKZEKA = 59, }; struct LLM_KV; diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cpp index cf0220367186..08c22b6ed823 100644 --- a/src/models/afmoe.cpp +++ b/src/models/afmoe.cpp @@ -14,9 +14,7 @@ void llama_model_afmoe::load_arch_hparams(llama_model_loader & ml) { // Pattern: 3 sliding - 1 full (global_attn_every_n_layers = 4) if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/bert.cpp b/src/models/bert.cpp index ca0281d306b1..9cc03dc56ad0 100644 --- a/src/models/bert.cpp +++ b/src/models/bert.cpp @@ -29,15 +29,13 @@ void llama_model_bert::load_arch_tensors(llama_model_loader &) { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_token_types}, TENSOR_NOT_REQUIRED); - if (arch == LLM_ARCH_BERT) { - pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train}, 0); + pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train}, 0); - cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED); - cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {n_embd}, TENSOR_NOT_REQUIRED); + cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED); + cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {n_embd}, TENSOR_NOT_REQUIRED); - cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - } + cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED); + cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {hparams.n_cls_out}, TENSOR_NOT_REQUIRED); tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0); tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0); @@ -53,20 +51,10 @@ void llama_model_bert::load_arch_tensors(llama_model_loader &) { layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0); layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}, 0); - if (hparams.moe_every_n_layers > 0 && i % hparams.moe_every_n_layers == 1) { - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff, n_expert}, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0); - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - } else { - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); - layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); - layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); - - if (arch == LLM_ARCH_NOMIC_BERT) { - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); - } - } + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, 0); layer.layer_out_norm_b = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}, 0); diff --git a/src/models/cohere2.cpp b/src/models/cohere2.cpp index e2b3662560df..7ad5244e427c 100644 --- a/src/models/cohere2.cpp +++ b/src/models/cohere2.cpp @@ -2,9 +2,7 @@ void llama_model_cohere2::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp index 5e02cd56e71d..7704cbb87299 100644 --- a/src/models/cohere2moe.cpp +++ b/src/models/cohere2moe.cpp @@ -25,12 +25,7 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) { } hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) { - hparams.set_swa_pattern(swa_period, true); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4, true); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cpp index b778b04145db..400d3c15051d 100644 --- a/src/models/deepseek2.cpp +++ b/src/models/deepseek2.cpp @@ -489,21 +489,12 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p const int ocr_rope_type = GGML_ROPE_TYPE_NEOX; GGML_ASSERT(n_embed_head == n_embd_head_k && n_embed_head == n_embd_head_v); - ggml_tensor * Qcur = NULL; - ggml_tensor * Kcur = NULL; - ggml_tensor * Vcur = NULL; - - Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur); - Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur); - Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embed_head, n_head, n_head, il); cb(Qcur, "q", il); cb(Kcur, "k", il); cb(Vcur, "v", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embed_head, n_head, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embed_head, n_head, n_tokens); - Vcur = ggml_reshape_3d(ctx0, Vcur, n_embed_head, n_head, n_tokens); - GGML_ASSERT(fabs(freq_base - 10000.0) < 1e-4); Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_embed_head, ocr_rope_type, 0, freq_base, 1, 0, 1, 0, 0); Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_embed_head, ocr_rope_type, 0, freq_base, 1, 0, 1, 0, 0); diff --git a/src/models/deepseek2ocr.cpp b/src/models/deepseek2ocr.cpp index 1c5c452e96d4..3d630699ef2d 100644 --- a/src/models/deepseek2ocr.cpp +++ b/src/models/deepseek2ocr.cpp @@ -40,9 +40,7 @@ void llama_model_deepseek2ocr::load_arch_tensors(llama_model_loader &) { for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; - layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0); - layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd}, 0); - layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd}, 0); + create_tensor_qkv(layer, i, n_embd, n_embd, n_embd, n_embd, 0); layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0); // norm diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 92b6272607b8..cd1194443c12 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -78,6 +78,19 @@ static ggml_tensor * wp_dflash_borrow_meta_mirrored( void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); + ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); + + hparams.llm_ffn_op = LLM_FFN_SILU; + std::string hidden_act; + if (ml.get_key(LLM_KV_HIDDEN_ACT, hidden_act, false)) { + if (hidden_act == "gelu" || hidden_act == "gelu_pytorch_tanh") { + hparams.llm_ffn_op = LLM_FFN_GELU; + } else if (hidden_act != "silu") { + throw std::runtime_error("unsupported DFlash hidden activation: " + hidden_act); + } + } + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale, false); hparams.f_final_logit_softcapping = 0.0f; @@ -195,9 +208,6 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { } // DSpark = DFlash + a semi-autoregressive Markov head and Confidence head - // - // TODO: only Qwen3-style backbones are supported for now; other backbones (e.g. Gemma4) - // need their own conversion path and graph tweaks const struct ggml_tensor * markov_meta = ml->get_tensor_meta("markov_w1.weight"); if (markov_meta) { const int64_t dspark_markov_rank = markov_meta->ne[0]; @@ -243,6 +253,9 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { // optional: reduced-vocab drafts ship their own lm head, full-vocab drafts can share the target's via ctx_other // a draft with its own embeddings + head references no target tensors and can run on devices the target does not use (e.g. -devd with a tensor-split target) output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab_draft }, TENSOR_NOT_REQUIRED); + if (output == nullptr && tok_embd != nullptr) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab_draft }, TENSOR_DUPLICATED); + } if (hparams.dsv4_hc_mult > 0) { const int64_t q_lora_rank = hparams.n_lora_q; @@ -312,7 +325,7 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0); layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0); - layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0); + layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, TENSOR_NOT_REQUIRED); layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0); layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), { n_embd_head_k }, 0); @@ -335,6 +348,11 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), { n_embd, n_gate_out }, 0); } + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED); + layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED); + layer.out_scale = create_tensor(tn(LLM_TENSOR_LAYER_OUT_SCALE, "weight", i), { 1 }, TENSOR_NOT_REQUIRED); + layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), { n_embd_head_k/2 }, TENSOR_NOT_REQUIRED | (i > 0 ? TENSOR_DUPLICATED : 0)); + // optional per-head attention sinks (e.g. Nemotron DSpark) layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), { n_head }, TENSOR_NOT_REQUIRED); @@ -355,21 +373,6 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { } } -std::unique_ptr<llm_graph_context> llama_model_dflash::build_arch_graph(const llm_graph_params & params) const { - switch (params.gtype) { - case LLM_GRAPH_TYPE_ENCODER: - return std::make_unique<graph<true>>(*this, params); - case LLM_GRAPH_TYPE_DEFAULT: - case LLM_GRAPH_TYPE_DECODER: - if (hparams.dsv4_hc_mult > 0) { - return std::make_unique<graph_dsv4>(*this, params); - } - return std::make_unique<graph<false>>(*this, params); - default: - GGML_ABORT("invalid graph type"); - }; -} - // MAD-LAB / upstream ggml-org#27310: the DFlash ENCODER BODY, factored out. // // `cur` is [n_embd_inp_enc, n_tokens] -- the raw target features, n_extract slices of @@ -1020,7 +1023,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra inp_attn = build_attn_inp_kv(); } - const float kq_scale = 1.0f/sqrtf(float(n_embd_head)); + const float kq_scale = hparams.f_attention_scale != 0.0f ? hparams.f_attention_scale : 1.0f/sqrtf(float(n_embd_head)); // drafts for M-RoPE targets use degenerate sections (temporal dim only) int sections[4]; @@ -1031,7 +1034,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra ? ggml_rope_multi(ctx0, cur, pos, nullptr, n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow) - : ggml_rope_ext(ctx0, cur, pos, nullptr, + : ggml_rope_ext(ctx0, cur, pos, model.layers[0].rope_freqs, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); }; @@ -1085,12 +1088,16 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra const auto & layer = model.layers[il]; ggml_tensor * Kcur = build_lora_mm(layer.wk, inp_g, layer.wk_s); - ggml_tensor * Vcur = build_lora_mm(layer.wv, inp_g, layer.wv_s); + const bool shared_kv = layer.wv == nullptr; + ggml_tensor * Vcur = shared_kv ? Kcur : build_lora_mm(layer.wv, inp_g, layer.wv_s); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il); + if (shared_kv) { + Vcur = ggml_rms_norm(ctx0, Vcur, hparams.f_norm_rms_eps); + } Kcur = build_rope(Kcur, inp_pos); cb(Kcur, "Kcur_injected", il); cb(Vcur, "Vcur_injected", il); @@ -1212,6 +1219,10 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra // return code and discards the result. ggml_build_forward_expand(gf, inp->tokens); } + + if (hparams.f_embedding_scale != 0.0f) { + inpL = ggml_scale(ctx0, inpL, hparams.f_embedding_scale); + } cb(inpL, "inp_noise_embd", -1); res->add_input(std::move(inp)); @@ -1231,7 +1242,8 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra ggml_tensor * Qcur = build_lora_mm(layer.wq, noise_norm, layer.wq_s); ggml_tensor * Kcur = build_lora_mm(layer.wk, noise_norm, layer.wk_s); - ggml_tensor * Vcur = build_lora_mm(layer.wv, noise_norm, layer.wv_s); + const bool shared_kv = layer.wv == nullptr; + ggml_tensor * Vcur = shared_kv ? Kcur : build_lora_mm(layer.wv, noise_norm, layer.wv_s); Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); @@ -1239,6 +1251,9 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra Qcur = build_norm(Qcur, layer.attn_q_norm, NULL, LLM_NORM_RMS, il); Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il); + if (shared_kv) { + Vcur = ggml_rms_norm(ctx0, Vcur, hparams.f_norm_rms_eps); + } Qcur = build_rope(Qcur, inp_pos); Kcur = build_rope(Kcur, inp_pos); @@ -1291,6 +1306,11 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra cb(cur, "attn_conv_out", il); } + if (layer.attn_post_norm) { + cur = build_norm(cur, layer.attn_post_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_post_norm", il); + } + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); cb(ffn_inp, "ffn_inp", il); @@ -1309,7 +1329,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra layer.ffn_gate, NULL, layer.ffn_gate_s, layer.ffn_down, NULL, layer.ffn_down_s, NULL, - LLM_FFN_SILU, LLM_FFN_PAR, il); + hparams.llm_ffn_op, LLM_FFN_PAR, il); cb(cur, "ffn_out", il); if (ffn_dynamic) { @@ -1317,7 +1337,15 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra cb(cur, "ffn_conv_out", il); } + if (layer.ffn_post_norm) { + cur = build_norm(cur, layer.ffn_post_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_post_norm", il); + } + cur = ggml_add(ctx0, cur, ffn_inp); + if (layer.out_scale) { + cur = ggml_mul(ctx0, cur, layer.out_scale); + } cb(cur, "l_out", il); inpL = cur; @@ -1647,3 +1675,18 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ } // MAD-LAB: end } + +std::unique_ptr<llm_graph_context> llama_model_dflash::build_arch_graph(const llm_graph_params & params) const { + switch (params.gtype) { + case LLM_GRAPH_TYPE_ENCODER: + return std::make_unique<graph<true>>(*this, params); + case LLM_GRAPH_TYPE_DEFAULT: + case LLM_GRAPH_TYPE_DECODER: + if (hparams.dsv4_hc_mult > 0) { + return std::make_unique<graph_dsv4>(*this, params); + } + return std::make_unique<graph<false>>(*this, params); + default: + GGML_ABORT("invalid graph type"); + }; +} diff --git a/src/models/eagle3.cpp b/src/models/eagle3.cpp index be466056df69..bfde35e43bd1 100644 --- a/src/models/eagle3.cpp +++ b/src/models/eagle3.cpp @@ -100,18 +100,6 @@ void llama_model_eagle3::load_arch_tensors(llama_model_loader &) { } } -std::unique_ptr<llm_graph_context> llama_model_eagle3::build_arch_graph(const llm_graph_params & params) const { - switch (params.gtype) { - case LLM_GRAPH_TYPE_ENCODER: - return std::make_unique<graph<true>>(*this, params); - case LLM_GRAPH_TYPE_DEFAULT: - case LLM_GRAPH_TYPE_DECODER: - return std::make_unique<graph<false>>(*this, params); - default: - GGML_ABORT("invalid graph type"); - }; -} - template <> ggml_tensor * llama_model_eagle3::graph<true>::build_inp_embd_enc() const { ggml_tensor * cur = nullptr; @@ -336,3 +324,15 @@ llama_model_eagle3::graph<false>::graph(const llama_model & model, const llm_gra ggml_build_forward_expand(gf, cur); } + +std::unique_ptr<llm_graph_context> llama_model_eagle3::build_arch_graph(const llm_graph_params & params) const { + switch (params.gtype) { + case LLM_GRAPH_TYPE_ENCODER: + return std::make_unique<graph<true>>(*this, params); + case LLM_GRAPH_TYPE_DEFAULT: + case LLM_GRAPH_TYPE_DECODER: + return std::make_unique<graph<false>>(*this, params); + default: + GGML_ABORT("invalid graph type"); + }; +} diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp index 976ee050adcd..840da5f67875 100644 --- a/src/models/exaone-moe.cpp +++ b/src/models/exaone-moe.cpp @@ -3,9 +3,7 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 128; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cpp index 9ba978956dc8..c1c55957a4fd 100644 --- a/src/models/exaone4.cpp +++ b/src/models/exaone4.cpp @@ -4,9 +4,7 @@ void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_layer() == 64) { // 32B hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/gemma-embedding.cpp b/src/models/gemma-embedding.cpp index 80ed3b1a4605..6c97883d8279 100644 --- a/src/models/gemma-embedding.cpp +++ b/src/models/gemma-embedding.cpp @@ -2,9 +2,7 @@ void llama_model_gemma_embedding::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC; - uint32_t swa_period = 6; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 6); hparams.causal_attn = false; // embeddings do not use causal attention diff --git a/src/models/gemma2.cpp b/src/models/gemma2.cpp index 2fbfb15a94a3..9e5243465543 100644 --- a/src/models/gemma2.cpp +++ b/src/models/gemma2.cpp @@ -3,9 +3,7 @@ void llama_model_gemma2::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; // default value of gemma 2 - uint32_t swa_period = 2; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 2); hparams.attn_soft_cap = true; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/gemma3.cpp b/src/models/gemma3.cpp index 690194529e38..f99bbaacd8ad 100644 --- a/src/models/gemma3.cpp +++ b/src/models/gemma3.cpp @@ -4,9 +4,7 @@ void llama_model_gemma3::load_arch_hparams(llama_model_loader & ml) { const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 6; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 6); ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); } else { diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp index 83eb8250aa94..4d47ddc62fc9 100644 --- a/src/models/gemma3n.cpp +++ b/src/models/gemma3n.cpp @@ -1,10 +1,8 @@ #include "models.h" void llama_model_gemma3n::load_arch_hparams(llama_model_loader & ml) { - uint32_t swa_period = 5; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 5); hparams.n_layer_kv_from_start = 20; hparams.f_attention_scale = 1.0f; @@ -82,7 +80,7 @@ std::unique_ptr<llm_graph_context> llama_model_gemma3n::build_arch_graph(const l } // get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim -static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { +static ggml_tensor * gemma3n_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { GGML_ASSERT(idx < (int) x->ne[2]); return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]), idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); @@ -139,7 +137,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * predictions = altup_predict(cur, il); // [n_embd, n_tokens, n_altup] // predicted value will go through self-attention and laurel - ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act); // [n_embd, n_tokens] + ggml_tensor * active_prediction = gemma3n_view_2d_slice(ctx0, predictions, i_altup_act); // [n_embd, n_tokens] cur = active_prediction; cb(cur, "active_prediction", il); @@ -176,7 +174,14 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par hparams.f_attention_scale, il); } else { // reuse KV cache of earlier layers - ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + ggml_tensor * Qcur; + if (model.layers[il].wqkv) { + ggml_tensor * qkv = build_lora_mm(model.layers[il].wqkv, cur); + const int64_t q_dim = n_embd_head * n_head; + Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv, q_dim, n_tokens, qkv->nb[1], 0)); + } else { + Qcur = build_lora_mm(model.layers[il].wq, cur); + } cb(Qcur, "Qcur", il); Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); @@ -229,13 +234,13 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * first_prediction; // [n_embd, n_tokens] { - first_prediction = ggml_view_2d_slice(ctx0, corrected, i_altup_act); // [n_embd, n_tokens] + first_prediction = gemma3n_view_2d_slice(ctx0, corrected, i_altup_act); // [n_embd, n_tokens] first_prediction = ggml_mul(ctx0, first_prediction, model.layers[il].altup_correct_scale); first_prediction = build_lora_mm(model.layers[il].per_layer_inp_gate, first_prediction); first_prediction = ggml_gelu(ctx0, first_prediction); // [n_embd_altup, n_tokens] cb(first_prediction, "first_prediction_gated", il); - ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_altup, n_tokens] + ggml_tensor * inp_this_layer = gemma3n_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_altup, n_tokens] first_prediction = ggml_mul(ctx0, first_prediction, inp_this_layer); // [n_embd_altup, n_tokens] cb(first_prediction, "first_prediction_scaled", il); @@ -246,7 +251,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par } // equivalent to python code: corrected_predictions[1:] += first_prediction { - ggml_tensor * slice_first = ggml_view_2d_slice(ctx0, corrected, 0); + ggml_tensor * slice_first = gemma3n_view_2d_slice(ctx0, corrected, 0); ggml_tensor * slice_rest = ggml_view_3d( ctx0, corrected, n_embd, n_tokens, n_altup - 1, ggml_row_size(corrected->type, n_embd), ggml_row_size(corrected->type, n_embd * n_tokens), n_embd * n_tokens * ggml_element_size(corrected)); @@ -264,7 +269,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par // cur now has multiple altup(s), we want to merge them back to 1 altup { - ggml_tensor * target_magnitude = calc_magnitude(ggml_view_2d_slice(ctx0, cur, i_altup_act)); // [n_embd, n_tokens] + ggml_tensor * target_magnitude = calc_magnitude(gemma3n_view_2d_slice(ctx0, cur, i_altup_act)); // [n_embd, n_tokens] // do a view to skip the first slice (active altup) ggml_tensor * alt_slice = ggml_view_3d(ctx0, cur, n_embd, n_tokens, n_altup - 1, ggml_row_size(cur->type, n_embd), @@ -276,9 +281,9 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par cb(altup_unembd, "altup_unembd", -1); // equivalent to torch.mean(hidden_states, dim=0) - cur = ggml_view_2d_slice(ctx0, cur, 0); // [n_embd, n_tokens] + cur = gemma3n_view_2d_slice(ctx0, cur, 0); // [n_embd, n_tokens] for (int i = 0; i < n_altup - 1; ++i) { - cur = ggml_add(ctx0, cur, ggml_view_2d_slice(ctx0, altup_unembd, i)); + cur = ggml_add(ctx0, cur, gemma3n_view_2d_slice(ctx0, altup_unembd, i)); } cur = ggml_scale(ctx0, cur, 1.0f / float(n_altup)); // [n_embd, n_tokens] cb(cur, "unembd_merged", -1); @@ -412,7 +417,7 @@ ggml_tensor * llama_model_gemma3n::graph::altup_compute_router_modalities(ggml_t // input cur shape: [n_embd, n_tokens, n_altup] // output shape: [n_embd, n_tokens, n_altup] ggml_tensor * llama_model_gemma3n::graph::altup_predict(ggml_tensor * cur, int il) { - ggml_tensor * activated = ggml_view_2d_slice(ctx0, cur, i_altup_act); // [n_embd, n_tokens] + ggml_tensor * activated = gemma3n_view_2d_slice(ctx0, cur, i_altup_act); // [n_embd, n_tokens] ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens] cb(modalities, "modalities", il); @@ -440,7 +445,7 @@ ggml_tensor * llama_model_gemma3n::graph::altup_correct(ggml_tensor * prediction ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens] cb(modalities, "modalities", il); - ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act); + ggml_tensor * active_prediction = gemma3n_view_2d_slice(ctx0, predictions, i_altup_act); ggml_tensor * innovation = ggml_sub(ctx0, activated, active_prediction); // [n_embd, n_tokens] cb(innovation, "innovation", il); diff --git a/src/models/gemma4-assistant.cpp b/src/models/gemma4-assistant.cpp index 8431ec2a1fc7..74d06151e35a 100644 --- a/src/models/gemma4-assistant.cpp +++ b/src/models/gemma4-assistant.cpp @@ -4,7 +4,7 @@ void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) { hparams.n_embd_inp_impl = hparams.n_embd_out(); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer_all); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); uint32_t n_kv_shared_layers = 0; ml.get_key(LLM_KV_ATTENTION_SHARED_KV_LAYERS, n_kv_shared_layers, false); diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp index 0cd95742d14a..67de74c5471b 100644 --- a/src/models/gemma4.cpp +++ b/src/models/gemma4.cpp @@ -2,7 +2,7 @@ void llama_model_gemma4::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); uint32_t n_kv_shared_layers = 0; ml.get_key(LLM_KV_ATTENTION_SHARED_KV_LAYERS, n_kv_shared_layers, false); @@ -75,9 +75,13 @@ void llama_model_gemma4::load_arch_tensors(llama_model_loader &) { layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); // note: use_alternative_attention (v_proj is optional, if it's not present, use k_proj) - layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head * n_head}, 0); - layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k}, kv_flags); - layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v}, TENSOR_NOT_REQUIRED); + layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), + {n_embd, n_embd_head * n_head + n_embd_k + n_embd_v}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); + if (!layer.wqkv) { + layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head * n_head}, 0); + layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k}, kv_flags); + layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v}, TENSOR_NOT_REQUIRED); + } layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head * n_head, n_embd}, 0); layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head}, 0); @@ -141,7 +145,7 @@ std::unique_ptr<llm_graph_context> llama_model_gemma4::build_arch_graph(const ll } // get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim -static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { +static ggml_tensor * gemma4_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { GGML_ASSERT(idx < (int) x->ne[2]); return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]), idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); @@ -202,9 +206,17 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para // Q projection (shared for both non-KV and KV layers) // this is to mirror Gemma4Attention in pytorch code + ggml_tensor * qkv_fused = nullptr; ggml_tensor * Qcur; - { + if (model.layers[il].wqkv) { + qkv_fused = build_lora_mm(model.layers[il].wqkv, cur, model.layers[il].wqkv_s); + cb(qkv_fused, "wqkv", il); + const int64_t q_dim = n_embd_head * n_head; + Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv_fused, q_dim, n_tokens, qkv_fused->nb[1], 0)); + } else { Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); + } + { cb(Qcur, "Qcur", il); Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); @@ -219,12 +231,22 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para // self-attention if (hparams.has_kv(il)) { - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s); + ggml_tensor * Kcur; + ggml_tensor * Vcur; + if (qkv_fused) { + const int64_t q_dim = n_embd_head * n_head; + const int64_t k_dim = n_embd_head * n_head_kv; + const int64_t v_dim = n_embd_head * n_head_kv; + const size_t esize = ggml_element_size(qkv_fused); + Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv_fused, k_dim, n_tokens, qkv_fused->nb[1], q_dim * esize)); + Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv_fused, v_dim, n_tokens, qkv_fused->nb[1], (q_dim + k_dim) * esize)); + } else { + Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s); + Vcur = model.layers[il].wv + ? build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s) + : Kcur; // if v_proj is not present, use Kcur as Vcur + } cb(Kcur, "Kcur", il); - - ggml_tensor * Vcur = model.layers[il].wv - ? build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s) - : Kcur; // if v_proj is not present, use Kcur as Vcur cb(Vcur, "Vcur", il); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); @@ -350,7 +372,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para cur = build_lora_mm(model.layers[il].per_layer_inp_gate, cur); // [n_embd_per_layer, n_tokens] cur = ggml_gelu(ctx0, cur); - ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens] + ggml_tensor * inp_this_layer = gemma4_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens] // TODO @ngxson : improve this if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) { diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp index 8a8f7e19ff08..c177ae78756e 100644 --- a/src/models/granite-hybrid.cpp +++ b/src/models/granite-hybrid.cpp @@ -30,7 +30,7 @@ void llama_model_granite_hybrid::load_arch_hparams(llama_model_loader & ml) { case 768: type = LLM_TYPE_350M; break; case 1536: type = (hparams.n_ff() == 512 ? LLM_TYPE_7B_A1B : LLM_TYPE_1B); break; case 2048: case 2560: type = LLM_TYPE_3B; break; - case 4096: type = LLM_TYPE_32B; break; + case 4096: type = LLM_TYPE_32B_A9B; break; default: type = LLM_TYPE_UNKNOWN; } diff --git a/src/models/granite-moe.cpp b/src/models/granite-moe.cpp index 09be49393e30..febe1bfa7aa4 100644 --- a/src/models/granite-moe.cpp +++ b/src/models/granite-moe.cpp @@ -8,8 +8,8 @@ void llama_model_granite_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); switch (hparams.n_layer()) { - case 32: type = LLM_TYPE_3B; break; - case 40: type = LLM_TYPE_3B; break; + case 24: type = LLM_TYPE_1B_A400M; break; + case 32: type = LLM_TYPE_3B_A800M; break; // Add additional layer/vocab/etc checks here for other model sizes default: type = LLM_TYPE_UNKNOWN; } diff --git a/src/models/granite.cpp b/src/models/granite.cpp index 9e9f97e94dca..60d463aedab0 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -38,7 +38,16 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) { switch (hparams.n_layer()) { case 32: type = LLM_TYPE_3B; break; - case 40: type = LLM_TYPE_3B; break; + case 40: { + switch (hparams.n_embd) { + case 2048: type = LLM_TYPE_2B; break; + case 2560: type = LLM_TYPE_3B; break; + case 4096: type = LLM_TYPE_8B; break; + default: type = LLM_TYPE_UNKNOWN; + } + break; + } + case 64: type = LLM_TYPE_30B; break; // Add additional layer/vocab/etc checks here for other model sizes default: type = LLM_TYPE_UNKNOWN; } diff --git a/src/models/hrm-text.cpp b/src/models/hrm-text.cpp new file mode 100644 index 000000000000..4a9a67b6c147 --- /dev/null +++ b/src/models/hrm-text.cpp @@ -0,0 +1,213 @@ +#include "models.h" + +// HRM-Text: alternating low/high transformer stacks over the same token stream. +// Reference: HrmTextModel in transformers, DFM Mimir 1B. + +void llama_model_hrm_text::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); + + ml.get_key(LLM_KV_HRM_LAYERS_PER_STACK, hparams.n_hrm_layers_per_stack); + ml.get_key(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles); + ml.get_key(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles); + + // prefix-LM prefill is not implemented (causal attention only); kept for round-trip + ml.get_key(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm, false); + + GGML_ASSERT(hparams.n_hrm_layers_per_stack > 0); + GGML_ASSERT(hparams.n_hrm_h_cycles > 0); + GGML_ASSERT(hparams.n_hrm_l_cycles > 0); + + // the GGUF block count is the expanded cache-slot count + const uint32_t n_slot = hparams.n_hrm_layers_per_stack * hparams.n_hrm_h_cycles * (hparams.n_hrm_l_cycles + 1); + GGML_ASSERT(hparams.n_layer() == n_slot); + + switch (hparams.n_embd) { + case 1536: + type = LLM_TYPE_1B; + break; + default: + type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_hrm_text::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); + + // output + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED); + } + + hrm_z_l_init = create_tensor(tn(LLM_TENSOR_HRM_Z_L_INIT), { n_embd }, 0); + + const int lps = hparams.n_hrm_layers_per_stack; + + // blocks [0, lps) hold the low stack, blocks [lps, 2*lps) hold the high stack. + // the first low and high passes create the layers; later passes alias them. + const int l_first = 0; + const int h_first = hparams.n_hrm_l_cycles * lps; + + for (int h = 0; h < (int) hparams.n_hrm_h_cycles; ++h) { + for (int l = 0; l < (int) hparams.n_hrm_l_cycles + 1; ++l) { + const int slot_base = (h * (hparams.n_hrm_l_cycles + 1) + l) * lps; + const int blk_base = l == (int) hparams.n_hrm_l_cycles ? lps : 0; + + if (h > 0 || (l > 0 && l < (int) hparams.n_hrm_l_cycles)) { + // alias pass: these cache slots hold the same layers as the first passes + const int src_base = l == (int) hparams.n_hrm_l_cycles ? h_first : l_first; + for (int il = 0; il < lps; ++il) { + layers[slot_base + il] = layers[src_base + il]; + } + continue; + } + + for (int il = 0; il < lps; ++il) { + auto & layer = layers[slot_base + il]; + const int bid = blk_base + il; + + create_tensor_qkv(layer, bid, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); + + // sigmoid attention gate, applied to the attention output before o_proj + layer.wqkv_gate = + create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", bid), { n_embd, n_embd_head_k * n_head }, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", bid), { n_embd_head_k * n_head, n_embd }, 0); + + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", bid), { n_embd, n_ff }, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", bid), { n_ff, n_embd }, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", bid), { n_embd, n_ff }, 0); + } + } + } +} + +std::unique_ptr<llm_graph_context> llama_model_hrm_text::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique<graph>(*this, params); +} + +// one stack invocation: lps pre-norm decoder layers, then the parameterless final norm +ggml_tensor * llama_model_hrm_text::graph::build_stack(llm_graph_input_attn_kv * inp_attn, + ggml_tensor * inp_pos, + ggml_tensor * cur, + int slot_base) const { + const float kq_scale = 1.0f / sqrtf(float(n_embd_head_k)); + + const int lps = model.hparams.n_hrm_layers_per_stack; + + for (int il = 0; il < lps; ++il) { + const int s = slot_base + il; + const auto & layer = model.layers[s]; + + ggml_tensor * inpSA = cur; + + cur = build_norm(cur, nullptr, nullptr, LLM_NORM_RMS, s); + cb(cur, "attn_norm", s); + + // sigmoid-gated self-attention (same shape as qwen3next attention layers) + { + ggml_tensor * gate = build_lora_mm(layer.wqkv_gate, cur); + cb(gate, "attn_gate_proj", s); + + auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head_k, n_head, n_head_kv, s); + + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur", s); + + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Kcur, "Kcur", s); + + cur = build_attn(inp_attn, + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, s); + cb(cur, "attn_pregate", s); + + gate = ggml_sigmoid(ctx0, gate); + cb(gate, "attn_gate_sigmoid", s); + + cur = ggml_mul(ctx0, cur, gate); + cb(cur, "attn_gated", s); + + cur = build_lora_mm(layer.wo, cur, layer.wo_s); + cb(cur, "attn_out", s); + } + + cur = ggml_add(ctx0, cur, inpSA); + cb(cur, "attn_add", s); + + inpSA = cur; + cur = build_norm(cur, nullptr, nullptr, LLM_NORM_RMS, s); + cb(cur, "ffn_norm", s); + + cur = build_ffn(cur, + layer.ffn_up, nullptr, nullptr, + layer.ffn_gate, nullptr, nullptr, + layer.ffn_down, nullptr, nullptr, + nullptr, + LLM_FFN_SILU, LLM_FFN_PAR, s); + cb(cur, "ffn_out", s); + + cur = ggml_add(ctx0, cur, inpSA); + cb(cur, "ffn_add", s); + + cur = build_cvec(cur, s); + cb(cur, "l_out", s); + } + + cur = build_norm(cur, nullptr, nullptr, LLM_NORM_RMS, slot_base); + cb(cur, "stack_norm", slot_base); + + return cur; +} + +llama_model_hrm_text::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params), + model(model) { + ggml_tensor * cur; + + // {n_embd, n_tokens}, scaled by hparams.f_embedding_scale inside build_inp_embd + ggml_tensor * zH = build_inp_embd(model.tok_embd); + + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // the learned low-cycle state is [n_embd]; binary ops broadcast it over [n_embd, n_tokens] + ggml_tensor * zL = model.hrm_z_l_init; + + for (uint32_t h = 0; h < model.hparams.n_hrm_h_cycles; ++h) { + for (uint32_t l = 0; l < model.hparams.n_hrm_l_cycles; ++l) { + const int slot_base = (h * (model.hparams.n_hrm_l_cycles + 1) + l) * model.hparams.n_hrm_layers_per_stack; + + zL = build_stack(inp_attn, inp_pos, ggml_add(ctx0, zH, zL), slot_base); + } + + const int slot_base = (h * (model.hparams.n_hrm_l_cycles + 1) + model.hparams.n_hrm_l_cycles) * + model.hparams.n_hrm_layers_per_stack; + + zH = build_stack(inp_attn, inp_pos, ggml_add(ctx0, zH, zL), slot_base); + } + + cur = zH; + + if (inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/hunyuan-vl.cpp b/src/models/hunyuan-vl.cpp index da9bb74de7eb..18b6eaf8cd51 100644 --- a/src/models/hunyuan-vl.cpp +++ b/src/models/hunyuan-vl.cpp @@ -83,6 +83,8 @@ llama_model_hunyuan_vl::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = inpL; + ggml_tensor * inpSA = inpL; // norm diff --git a/src/models/jais2.cpp b/src/models/jais2.cpp index 8610fcc9f82f..64813b7b6b23 100644 --- a/src/models/jais2.cpp +++ b/src/models/jais2.cpp @@ -29,15 +29,9 @@ void llama_model_jais2::load_arch_tensors(llama_model_loader &) { layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0); - layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0); - layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0); - layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0); + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); - // attention biases - all have shape n_embd (output dimension of projections) - layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, 0); - layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd}, 0); - layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd}, 0); layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0); layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); diff --git a/src/models/jina-bert-v3.cpp b/src/models/jina-bert-v3.cpp index 1c974a6f16cc..78cf9d835976 100644 --- a/src/models/jina-bert-v3.cpp +++ b/src/models/jina-bert-v3.cpp @@ -19,16 +19,6 @@ void llama_model_jina_bert_v3::load_arch_tensors(llama_model_loader &) { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_token_types}, TENSOR_NOT_REQUIRED); - if (arch == LLM_ARCH_BERT) { - pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train}, 0); - - cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED); - cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {n_embd}, TENSOR_NOT_REQUIRED); - - cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - } - tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0); tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0); @@ -43,20 +33,10 @@ void llama_model_jina_bert_v3::load_arch_tensors(llama_model_loader &) { layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0); layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}, 0); - if (hparams.moe_every_n_layers > 0 && i % hparams.moe_every_n_layers == 1) { - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff, n_expert}, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0); - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - } else { - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); - layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); - layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); - - if (arch == LLM_ARCH_NOMIC_BERT) { - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); - } - } + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, 0); layer.layer_out_norm_b = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}, 0); diff --git a/src/models/kimi-k3.cpp b/src/models/kimi-k3.cpp index e6b9a53ac8f9..112b0984903b 100644 --- a/src/models/kimi-k3.cpp +++ b/src/models/kimi-k3.cpp @@ -1,4 +1,6 @@ #include "models.h" + +#include <algorithm> #include "llama-memory-recurrent.h" // @@ -357,7 +359,8 @@ static ggml_tensor * kimi_k3_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, int64_t d_conv, int64_t head_dim, int64_t n_head, - int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head) { + int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head, + int64_t mem_size, int64_t K_rs) { const int64_t d_inner = head_dim * n_head; const int64_t conv_state_size = (d_conv - 1) * d_inner; const int64_t n_embd_r_total = 3 * conv_state_size; @@ -371,14 +374,19 @@ static ggml_tensor * kimi_k3_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs); ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0); - ggml_tensor * last_conv_x = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, - conv_x->nb[1], conv_x->nb[2], n_seq_tokens * conv_x->nb[0]); - ggml_build_forward_expand(gf, - ggml_cpy(ctx0, last_conv_x, - ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs, - (d_conv - 1) * ggml_element_size(conv_states_all), - n_embd_r_total * ggml_element_size(conv_states_all), - (kv_head * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); + // group s holds the conv window s tokens back. + // [TAG_RECURRENT_ROLLBACK_SPLITS]: the last K_rs tokens must share one ubatch. + for (int64_t s = 0; s < K_rs; ++s) { + const int64_t s_idx = std::max<int64_t>(0, n_seq_tokens - s); + ggml_tensor * conv_x_s = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, + conv_x->nb[1], conv_x->nb[2], s_idx * conv_x->nb[0]); + ggml_build_forward_expand(gf, + ggml_cpy(ctx0, conv_x_s, + ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs, + (d_conv - 1) * ggml_element_size(conv_states_all), + n_embd_r_total * ggml_element_size(conv_states_all), + ((s * mem_size + kv_head) * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); + } ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner); ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight); @@ -399,9 +407,12 @@ ggml_tensor * llama_model_kimi_k3::graph::build_kda_layer( ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs); - ggml_tensor * Qcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head); - ggml_tensor * Kcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head); - ggml_tensor * Vcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head); + const int64_t mem_size = mctx_cur->get_size(); + const int64_t K_rs = (int64_t) cparams.n_rs_seq + 1; + + ggml_tensor * Qcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); + ggml_tensor * Kcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); + ggml_tensor * Vcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); cb(Qcur, "kda_q_conv", il); cb(Kcur, "kda_k_conv", il); cb(Vcur, "kda_v_conv", il); @@ -441,20 +452,13 @@ ggml_tensor * llama_model_kimi_k3::graph::build_kda_layer( ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs); state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head_kda, n_seqs); - const float eps = hparams.f_norm_rms_eps; - Qcur = build_gdn_l2_norm(ctx0, Qcur, eps); - Kcur = build_gdn_l2_norm(ctx0, Kcur, eps); + const float eps_norm = hparams.f_norm_rms_eps; + Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm); + Kcur = build_gdn_l2_norm(ctx0, Kcur, eps_norm); - auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il); - - ggml_tensor * output = ggml_cont(ctx0, attn_out.first); + ggml_tensor * output = build_recurrent_attn(inp_rs, ssm_states_all, Qcur, Kcur, Vcur, g1, beta, state, il); + output = ggml_cont(ctx0, output); cb(output, "kda_scan_out", il); - ggml_tensor * new_state = attn_out.second; - - ggml_build_forward_expand(gf, - ggml_cpy(ctx0, new_state, - ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs, - kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all)))); // K3: single full-rank gate (kimi-linear factors this as g_b(g_a(x))) ggml_tensor * cur_2d = ggml_reshape_2d(ctx0, cur_3d, cur_3d->ne[0], n_seq_tokens * n_seqs); diff --git a/src/models/kimi-linear.cpp b/src/models/kimi-linear.cpp index 3c1b2e69a1b7..b9cf28d85cf0 100644 --- a/src/models/kimi-linear.cpp +++ b/src/models/kimi-linear.cpp @@ -195,7 +195,7 @@ static ggml_tensor * causal_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_t // Causal Conv1d function for Q,K,V // When qkv is 0, it is Q, 1 is K, 2 is V // Step 1: Q, K, V projections -> [d_inner, n_tokens] - ggml_tensor * x_proj = ggml_mul_mat(ctx0, proj_w, x); + ggml_tensor * x_proj = proj_w ? ggml_mul_mat(ctx0, proj_w, x) : x; // Reshape input: {d_inner, n_tokens} -> {d_inner, n_seq_tokens, n_seqs} ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs); @@ -295,9 +295,20 @@ llama_model_kimi_linear::graph::graph(const llama_model & model, const llm_graph ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); cb(conv_states_all, "conv_states_all", il); ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs); - ggml_tensor * Qcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); - ggml_tensor * Kcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); - ggml_tensor * Vcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); + ggml_tensor * q_in = cur, * k_in = cur, * v_in = cur; + ggml_tensor * q_w = layer.wq, * k_w = layer.wk, * v_w = layer.wv; + if (layer.wqkv) { + ggml_tensor * qkv = ggml_mul_mat(ctx0, layer.wqkv, cur); + const int64_t d_inner = head_dim * n_head; + const size_t esize = ggml_element_size(qkv); + q_in = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv, d_inner, n_tokens, qkv->nb[1], 0)); + k_in = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv, d_inner, n_tokens, qkv->nb[1], d_inner * esize)); + v_in = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv, d_inner, n_tokens, qkv->nb[1], 2 * d_inner * esize)); + q_w = nullptr; k_w = nullptr; v_w = nullptr; + } + ggml_tensor * Qcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, q_in, q_w, layer.ssm_q_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); + ggml_tensor * Kcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, k_in, k_w, layer.ssm_k_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); + ggml_tensor * Vcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, v_in, v_w, layer.ssm_v_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); // g1 = -exp(A_log) * softplus(f_b(f_a(x)) + dt_bias) ggml_tensor * f_a = ggml_mul_mat(ctx0, layer.ssm_f_a, cur); @@ -331,6 +342,7 @@ llama_model_kimi_linear::graph::graph(const llama_model & model, const llm_graph ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs); state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head, n_seqs); + const float eps_norm = hparams.f_norm_rms_eps; Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm); diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp index 78afa133aaf5..eebd6eb42178 100644 --- a/src/models/laguna.cpp +++ b/src/models/laguna.cpp @@ -36,9 +36,7 @@ void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, /*dense_first=*/true); // XS.2: FULL at il%4==0 + load_swa_pattern(ml, 4, /*dense_first=*/true); // XS.2: FULL at il%4==0 // Per-layer-type RoPE: full layers use YaRN θ=500000 over 64 dims; // SWA layers use default RoPE θ=10000 over 128 dims. Base load_hparams diff --git a/src/models/llada.cpp b/src/models/llada.cpp index 87d4259f9a74..ae3d6925c136 100644 --- a/src/models/llada.cpp +++ b/src/models/llada.cpp @@ -36,12 +36,7 @@ void llama_model_llada::load_arch_tensors(llama_model_loader &) { layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0); - // Use separate Q, K, V projections without bias, matching LLaDALlamaBlock - layer.wq = - create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0); - layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0); - layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0); - // No bias for QKV projections as per config: include_bias=false, include_qkv_bias=false + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0); layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), { n_embd }, TENSOR_NOT_REQUIRED); diff --git a/src/models/llama4.cpp b/src/models/llama4.cpp index 8a812beffac4..4f79b4d1eefe 100644 --- a/src/models/llama4.cpp +++ b/src/models/llama4.cpp @@ -16,9 +16,7 @@ void llama_model_llama4::load_arch_hparams(llama_model_loader & ml) { hparams.f_attn_temp_scale = 0.1f; hparams.f_attn_temp_offset = 1.0f; - uint32_t swa_period = 4; // pattern: 3 chunked - 1 full - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); // pattern: 3 chunked - 1 full hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/mamba-base.cpp b/src/models/mamba-base.cpp index 03ee3805bf80..370de0f0141d 100644 --- a/src/models/mamba-base.cpp +++ b/src/models/mamba-base.cpp @@ -100,6 +100,8 @@ ggml_tensor * llm_build_mamba_base::build_mamba_layer(llm_graph_input_rs * inp, dt = build_norm(dt, layer.ssm_dt_norm, NULL, LLM_NORM_RMS, il); B = build_norm(B, layer.ssm_b_norm, NULL, LLM_NORM_RMS, il); C = build_norm(C, layer.ssm_c_norm, NULL, LLM_NORM_RMS, il); + } else { + dt = ggml_cont(ctx0, dt); } // {dt_rank, d_inner} @ {dt_rank, n_seq_tokens, n_seqs} => {d_inner, n_seq_tokens, n_seqs} diff --git a/src/models/maple.cpp b/src/models/maple.cpp new file mode 100644 index 000000000000..7604b7dfee45 --- /dev/null +++ b/src/models/maple.cpp @@ -0,0 +1,150 @@ +#include "models.h" + +void llama_model_maple::load_arch_hparams(llama_model_loader & ml) { + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); + + ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all); + + switch (hparams.n_layer()) { + case 24: type = LLM_TYPE_20B; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_maple::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t n_ff_exp = hparams.n_ff_exp(); + const int64_t head_dim = hparams.n_embd_head_k(); + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + if (n_expert == 0) { + throw std::runtime_error("n_expert must be > 0 for Maple"); + } + if (n_expert_used == 0) { + throw std::runtime_error("n_expert_used must be > 0 for Maple"); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + create_tensor_qkv(layer, i, n_embd, n_head * head_dim, n_head_kv * head_dim, n_head_kv * head_dim, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * head_dim, n_embd}, 0); + + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {head_dim}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {head_dim}, 0); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + } +} + +std::unique_ptr<llm_graph_context> llama_model_maple::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique<graph>(*this, params); +} + +llama_model_maple::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_k(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_v()); + + ggml_tensor * inpL = build_inp_embd(model.tok_embd); + ggml_tensor * inp_pos = build_inp_pos(); + auto * inp_attn = build_attn_inp_kv_iswa(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + ggml_tensor * cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + { + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); + + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + cb(Kcur, "Kcur_normed", il); + + if (hparams.is_swa(il)) { + const int64_t n_rot_l = hparams.n_rot(il); + const float freq_base_l = model.get_rope_freq_base(cparams, il); + const float freq_scale_l = model.get_rope_freq_scale(cparams, il); + + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, + freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, + freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); + } + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, + model.layers[il].wo, nullptr, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il); + cb(cur, "attn_out", il); + } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, true, + 1.0f, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, + il); + cb(cur, "ffn_moe_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + ggml_tensor * cur = build_norm(inpL, model.output_norm, nullptr, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/mellum.cpp b/src/models/mellum.cpp index 872a9c8f556f..3ab79747af27 100644 --- a/src/models/mellum.cpp +++ b/src/models/mellum.cpp @@ -8,13 +8,7 @@ void llama_model_mellum::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - const auto res = ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - if (res) { - hparams.set_swa_pattern(swa_period); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cpp index 8772319f4611..a466984af01c 100644 --- a/src/models/mimo2.cpp +++ b/src/models/mimo2.cpp @@ -9,7 +9,7 @@ void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); float value_scale = 0.0f; if (ml.get_key(LLM_KV_ATTENTION_VALUE_SCALE, value_scale, false) && value_scale != 1.0f) { diff --git a/src/models/minimax-01.cpp b/src/models/minimax-01.cpp index 361114acc327..9fa2e8fc01cd 100644 --- a/src/models/minimax-01.cpp +++ b/src/models/minimax-01.cpp @@ -229,6 +229,7 @@ llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_ ggml_set_input(inp->inp_k_decay); cb(inp->inp_k_decay, "k_decay_exp", -1); + // [TAG_RESERVE_DIAG_DECAY] inp->inp_diag_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_seq_tokens, n_seq_tokens, n_head, n_seqs); ggml_set_input(inp->inp_diag_decay); cb(inp->inp_diag_decay, "diag_decay_exp", -1); diff --git a/src/models/minimax-m2.cpp b/src/models/minimax-m2.cpp index c2e69bfaa89f..7a22af036bc7 100644 --- a/src/models/minimax-m2.cpp +++ b/src/models/minimax-m2.cpp @@ -71,14 +71,13 @@ llama_model_minimax_m2::graph::graph(const llama_model & model, const llm_graph_ cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); - // compute Q and K and RoPE them - ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, n_head, + n_embd_head, n_head_kv, + n_embd_head, n_head_kv, + il, false); cb(Qcur, "Qcur", il); - - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); cb(Kcur, "Kcur", il); - - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 80260a6295be..53caca1a5c65 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -23,7 +23,12 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k); ml.get_key(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, hparams.indexer_block_size); ml.get_key(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks); - msa_p = { (int) hparams.indexer_block_size, (int) hparams.indexer_top_k, (int) hparams.indexer_local_blocks }; + + msa_p = { + /*.blk =*/ (int) hparams.indexer_block_size, + /*.topk_blocks =*/ (int) hparams.indexer_top_k, + /*.local =*/ (int) hparams.indexer_local_blocks, + }; GGML_ASSERT(hparams.indexer_block_size > 0); // avoid div by zero @@ -191,7 +196,7 @@ ggml_tensor * llama_model_minimax_m3::graph::build_attn_msa_fa( ggml_tensor * o = ggml_flash_attn_ext(ctx0, q, k, v, mask, kq_scale, hparams.f_max_alibi_bias, 0.0f); - ggml_flash_attn_ext_set_prec(o, GGML_PREC_F32); + ggml_prec_set_acc(o, GGML_PREC_F32); cb(o, "msa_fattn", il); // [D, Gp, R, C] -> [D, Gp, C, R] -> [n_embd, T] @@ -389,7 +394,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * iq4 = ggml_reshape_4d(ctx0, iq, n_idx_dim, Hd, 1, ns); ggml_tensor * sc = ggml_mul_mat(ctx0, ggml_reshape_4d(ctx0, ikp, n_idx_dim, n_ps, 1, ns), iq4); - ggml_mul_mat_set_prec(sc, GGML_PREC_F32); + ggml_prec_set_acc(sc, GGML_PREC_F32); // unmapped positions come out -inf, so they can never rank into the top-k sc = ggml_add_inplace(ctx0, sc, ggml_reshape_4d(ctx0, msa->pos_mask, n_ps, 1, 1, ns)); @@ -471,7 +476,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * sc = ggml_mul_mat(ctx0, ikp, ggml_reshape_2d(ctx0, iq_s, n_idx_dim, Hd*n_tps)); // indexer scores run in F32 - ggml_mul_mat_set_prec(sc, GGML_PREC_F32); + ggml_prec_set_acc(sc, GGML_PREC_F32); sc = ggml_reshape_3d(ctx0, sc, n_ps, Hd, n_tps); // unmapped positions (holes, padding, empty cells) come out -inf sc = ggml_add_inplace(ctx0, sc, pm_s); diff --git a/src/models/models.h b/src/models/models.h index 7782e3925b3a..1663c9a63011 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -976,6 +976,19 @@ struct llama_model_mamba2 : public llama_model_base { }; +struct llama_model_maple : public llama_model_base { + llama_model_maple(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_jamba : public llama_model_base { llama_model_jamba(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2010,6 +2023,27 @@ struct llama_model_plm : public llama_model_base { }; +struct llama_model_hrm_text : public llama_model_base { + llama_model_hrm_text(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + + const llama_model & model; + + ggml_tensor * build_stack( + llm_graph_input_attn_kv * inp_attn, + ggml_tensor * inp_pos, + ggml_tensor * cur, + int slot_base) const; + }; + + std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_bailingmoe : public llama_model_base { llama_model_bailingmoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2880,3 +2914,16 @@ struct llama_model_step35 : public llama_model_base { std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override; }; + + +struct llama_model_spark2_5 : public llama_model_base { + llama_model_spark2_5(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override; +}; diff --git a/src/models/modern-bert.cpp b/src/models/modern-bert.cpp index f3e9407e0125..b7542d59bd54 100644 --- a/src/models/modern-bert.cpp +++ b/src/models/modern-bert.cpp @@ -5,9 +5,7 @@ void llama_model_modern_bert::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC; ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - uint32_t swa_period = 3; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, true); + load_swa_pattern(ml, 3, true); } else { hparams.swa_type = LLAMA_SWA_TYPE_NONE; } diff --git a/src/models/muse-glimmer.cpp b/src/models/muse-glimmer.cpp index 0e94153088a9..0e5f75ebed9e 100644 --- a/src/models/muse-glimmer.cpp +++ b/src/models/muse-glimmer.cpp @@ -10,12 +10,7 @@ void llama_model_muse_glimmer::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) { - hparams.set_swa_pattern(swa_period); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4); switch (hparams.n_layer()) { case 52: type = LLM_TYPE_30B; break; diff --git a/src/models/nemotron-h-moe.cpp b/src/models/nemotron-h-moe.cpp index 4d03f49e0f8a..b4fb254300a8 100644 --- a/src/models/nemotron-h-moe.cpp +++ b/src/models/nemotron-h-moe.cpp @@ -100,6 +100,18 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llama_model & model, cons ggml_tensor * router_logits = build_lora_mm(layer.ffn_gate_inp, cur); cb(router_logits, "mtp_ffn_moe_logits", il); + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, NULL, layer.ffn_up_shexp_s, + NULL, NULL, NULL, + layer.ffn_down_shexp, NULL, layer.ffn_down_shexp_s, + NULL, + LLM_FFN_RELU_SQR, LLM_FFN_PAR, il); + cb(ffn_shexp, "mtp_ffn_shexp", il); + + if (layer.ffn_latent_down) { + cur = ggml_mul_mat(ctx0, layer.ffn_latent_down, cur); + } + ggml_tensor * moe_out = build_moe_ffn(cur, layer.ffn_gate_inp, @@ -118,13 +130,9 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llama_model & model, cons layer.ffn_down_exps_s); cb(moe_out, "mtp_ffn_moe_out", il); - ggml_tensor * ffn_shexp = build_ffn(cur, - layer.ffn_up_shexp, NULL, layer.ffn_up_shexp_s, - NULL, NULL, NULL, - layer.ffn_down_shexp, NULL, layer.ffn_down_shexp_s, - NULL, - LLM_FFN_RELU_SQR, LLM_FFN_PAR, il); - cb(ffn_shexp, "mtp_ffn_shexp", il); + if (layer.ffn_latent_up) { + moe_out = ggml_mul_mat(ctx0, layer.ffn_latent_up, moe_out); + } cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "mtp_ffn_out", il); diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp index d2c48f125ee2..be27650b061a 100644 --- a/src/models/nemotron-h.cpp +++ b/src/models/nemotron-h.cpp @@ -15,8 +15,10 @@ void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) { hparams.is_recr_impl[i] = i < hparams.n_layer() && hparams.n_head_kv(i) == 0 && hparams.n_ff(i) == 0; } - ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); - ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); // MTP head final_layernorm + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); // MTP head final_layernorm + if (!ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps, false)) { + hparams.f_norm_rms_eps = hparams.f_norm_eps; + } // Puzzle models set a different expert FFN size per layer ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); @@ -145,8 +147,14 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) { const int64_t n_head_i = hparams.n_head(i); const int64_t n_embd_k_gqa_i = hparams.n_embd_k_gqa(i); const int64_t n_embd_v_gqa_i = hparams.n_embd_v_gqa(i); - const int64_t n_ff_exp = hparams.n_ff_exp(i) ? (int64_t)hparams.n_ff_exp(i) : n_ff / (int64_t)hparams.n_expert_used(i); - const int64_t n_ff_shexp = hparams.n_ff_shexp; + const int64_t n_expert_used_i = hparams.n_expert_used(i); + const int64_t n_ff_exp_i = hparams.n_ff_exp(i); + if (n_ff_exp_i == 0 && n_expert_used_i == 0) { + throw std::runtime_error(format("%s: layer %d declares neither expert_feed_forward_length nor expert_used_count, " + "cannot determine the expert FFN size", __func__, i)); + } + const int64_t n_ff_exp = n_ff_exp_i ? n_ff_exp_i : n_ff / n_expert_used_i; + const int64_t n_ff_shexp = hparams.n_ff_shexp; // NextN input-fusion tensors layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, mtp_flags); @@ -164,6 +172,8 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) { layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, mtp_flags); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, mtp_flags); layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, mtp_flags); + layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, moe_n_embd}, mtp_flags | TENSOR_NOT_REQUIRED); + layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {moe_n_embd, n_embd}, mtp_flags | TENSOR_NOT_REQUIRED); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, moe_n_embd, n_expert}, mtp_flags); layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp, n_expert}, mtp_flags); layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, mtp_flags); diff --git a/src/models/nomic-bert-moe.cpp b/src/models/nomic-bert-moe.cpp index da4b62919bb9..924af5e0bc40 100644 --- a/src/models/nomic-bert-moe.cpp +++ b/src/models/nomic-bert-moe.cpp @@ -4,12 +4,10 @@ void llama_model_nomic_bert_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); ml.get_key(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers, 0); - if (hparams.n_layer() == 12 && hparams.n_embd == 768) { - if (arch == LLM_ARCH_NOMIC_BERT) { - type = LLM_TYPE_137M; - } else if (arch == LLM_ARCH_NOMIC_BERT_MOE && hparams.moe_every_n_layers == 2) { - type = LLM_TYPE_475M; - } + switch (hparams.n_layer()) { + case 12: + type = LLM_TYPE_475M; break; + default: type = LLM_TYPE_UNKNOWN; } } @@ -22,16 +20,6 @@ void llama_model_nomic_bert_moe::load_arch_tensors(llama_model_loader &) { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_token_types}, TENSOR_NOT_REQUIRED); - if (arch == LLM_ARCH_BERT) { - pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train}, 0); - - cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED); - cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {n_embd}, TENSOR_NOT_REQUIRED); - - cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - } - tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0); tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0); @@ -55,10 +43,6 @@ void llama_model_nomic_bert_moe::load_arch_tensors(llama_model_loader &) { layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); - - if (arch == LLM_ARCH_NOMIC_BERT) { - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); - } } layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, 0); diff --git a/src/models/nomic-bert.cpp b/src/models/nomic-bert.cpp index e7fc72286a6d..509787286efa 100644 --- a/src/models/nomic-bert.cpp +++ b/src/models/nomic-bert.cpp @@ -1,15 +1,12 @@ #include "models.h" void llama_model_nomic_bert::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); - ml.get_key(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers, 0); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); - if (hparams.n_layer() == 12 && hparams.n_embd == 768) { - if (arch == LLM_ARCH_NOMIC_BERT) { - type = LLM_TYPE_137M; - } else if (arch == LLM_ARCH_NOMIC_BERT_MOE && hparams.moe_every_n_layers == 2) { - type = LLM_TYPE_475M; - } + switch (hparams.n_layer()) { + case 12: + type = LLM_TYPE_137M; break; + default: type = LLM_TYPE_UNKNOWN; } } @@ -22,16 +19,6 @@ void llama_model_nomic_bert::load_arch_tensors(llama_model_loader &) { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_token_types}, TENSOR_NOT_REQUIRED); - if (arch == LLM_ARCH_BERT) { - pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train}, 0); - - cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED); - cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {n_embd}, TENSOR_NOT_REQUIRED); - - cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - } - tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0); tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0); @@ -46,20 +33,12 @@ void llama_model_nomic_bert::load_arch_tensors(llama_model_loader &) { layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0); layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}, 0); - if (hparams.moe_every_n_layers > 0 && i % hparams.moe_every_n_layers == 1) { - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff, n_expert}, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0); - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - } else { - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); - layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); - layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); - if (arch == LLM_ARCH_NOMIC_BERT) { - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); - } - } + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, 0); layer.layer_out_norm_b = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}, 0); diff --git a/src/models/olmo2.cpp b/src/models/olmo2.cpp index cb52cdef7204..fe5dc88d4469 100644 --- a/src/models/olmo2.cpp +++ b/src/models/olmo2.cpp @@ -6,9 +6,7 @@ void llama_model_olmo2::load_arch_hparams(llama_model_loader & ml) { const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = 1.0; // See olmo2.cpp @@ -93,14 +91,13 @@ llama_model_olmo2::graph<iswa>::graph(const llama_model & model, const llm_graph // self_attention { - // compute Q and K and RoPE them - ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, n_head, + n_embd_head, n_head_kv, + n_embd_head, n_head_kv, + il, false); cb(Qcur, "Qcur", il); - - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); cb(Kcur, "Kcur", il); - - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, diff --git a/src/models/olmoe.cpp b/src/models/olmoe.cpp index 1e2baeb207ff..11c53f3f4c9c 100644 --- a/src/models/olmoe.cpp +++ b/src/models/olmoe.cpp @@ -79,14 +79,13 @@ llama_model_olmoe::graph::graph(const llama_model & model, const llm_graph_param // self_attention { - // compute Q and K and RoPE them - ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, n_head, + n_embd_head, n_head_kv, + n_embd_head, n_head_kv, + il, false); cb(Qcur, "Qcur", il); - - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); cb(Kcur, "Kcur", il); - - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp index c9f9b677d06e..39660a325dd4 100644 --- a/src/models/openai-moe.cpp +++ b/src/models/openai-moe.cpp @@ -6,9 +6,7 @@ void llama_model_openai_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 2; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 2); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/plamo2.cpp b/src/models/plamo2.cpp index d946b3cff6da..ba1cea1465fd 100644 --- a/src/models/plamo2.cpp +++ b/src/models/plamo2.cpp @@ -142,6 +142,11 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para cur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, model, il); } + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + residual = ggml_get_rows(ctx0, residual, inp_out_ids); + } + // post_mixer_norm cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_post_norm", il); @@ -167,11 +172,6 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il); cb(cur, "ffn_post_norm", il); - if (il == n_layer - 1 && inp_out_ids) { - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - residual = ggml_get_rows(ctx0, residual, inp_out_ids); - } - // residual connection cur = ggml_add(ctx0, cur, residual); cb(cur, "ffn_residual", il); diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cpp index 16d0b1dcef71..f8235f8a8323 100644 --- a/src/models/plamo3.cpp +++ b/src/models/plamo3.cpp @@ -6,9 +6,7 @@ void llama_model_plamo3::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - uint32_t swa_period = 8; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 8); } else { hparams.swa_type = LLAMA_SWA_TYPE_NONE; } diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 7ac92e96639f..36b12e0b3715 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -541,8 +541,14 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( // Order: joint QG projection, QG split, Q norm, KV projection, K norm, RoPE, attention // Qwen3Next uses a single Q projection that outputs query + gate - ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ] + auto [Qcur_full, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head * 2, n_head, + n_embd_head, n_head_kv, + n_embd_head, n_head_kv, + il, false); cb(Qcur_full, "Qcur_full", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens, ggml_element_size(Qcur_full) * n_embd_head * 2, @@ -553,12 +559,6 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s); - cb(Kcur, "Kcur", il); - - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s); - cb(Vcur, "Vcur", il); - // Apply K normalization Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); @@ -719,6 +719,7 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); @@ -903,7 +904,11 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "mtp_attn_norm", il); - ggml_tensor * Qcur_full = build_lora_mm(layer.wq, cur, layer.wq_s); + auto [Qcur_full, Kcur, Vcur] = build_qkv(layer, cur, + n_embd_head * 2, n_head, + n_embd_head, n_head_kv, + n_embd_head, n_head_kv, + il, false); cb(Qcur_full, "mtp_Qcur_full", il); ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, @@ -922,12 +927,10 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens); cb(gate, "mtp_gate", il); - ggml_tensor * Kcur = build_lora_mm(layer.wk, cur, layer.wk_s); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il); cb(Kcur, "mtp_Kcur_normed", il); - ggml_tensor * Vcur = build_lora_mm(layer.wv, cur, layer.wv_s); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); cb(Vcur, "mtp_Vcur", il); diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp index 8843c91acfa2..9cf19b365087 100644 --- a/src/models/qwen35moe.cpp +++ b/src/models/qwen35moe.cpp @@ -361,8 +361,14 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( // Order: joint QG projection, QG split, Q norm, KV projection, K norm, RoPE, attention // Qwen3Next uses a single Q projection that outputs query + gate - ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ] + auto [Qcur_full, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head * 2, n_head, + n_embd_head, n_head_kv, + n_embd_head, n_head_kv, + il, false); cb(Qcur_full, "Qcur_full", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens, ggml_element_size(Qcur_full) * n_embd_head * 2, @@ -373,12 +379,6 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s); - cb(Kcur, "Kcur", il); - - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s); - cb(Vcur, "Vcur", il); - // Apply K normalization Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); @@ -521,6 +521,7 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); @@ -711,7 +712,11 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "mtp_attn_norm", il); - ggml_tensor * Qcur_full = build_lora_mm(layer.wq, cur, layer.wq_s); + auto [Qcur_full, Kcur, Vcur] = build_qkv(layer, cur, + n_embd_head * 2, n_head, + n_embd_head, n_head_kv, + n_embd_head, n_head_kv, + il, false); cb(Qcur_full, "mtp_Qcur_full", il); ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, @@ -730,12 +735,10 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens); cb(gate, "mtp_gate", il); - ggml_tensor * Kcur = build_lora_mm(layer.wk, cur, layer.wk_s); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il); cb(Kcur, "mtp_Kcur_normed", il); - ggml_tensor * Vcur = build_lora_mm(layer.wv, cur, layer.wv_s); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); cb(Vcur, "mtp_Vcur", il); diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp index c74fa9a07e7e..48572f1b6695 100644 --- a/src/models/qwen3next.cpp +++ b/src/models/qwen3next.cpp @@ -250,8 +250,14 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn( // Order: joint QG projection, QG split, Q norm, KV projection, K norm, RoPE, attention // Qwen3Next uses a single Q projection that outputs query + gate - ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); + auto [Qcur_full, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head * 2, n_head, + n_embd_head, n_head_kv, + n_embd_head, n_head_kv, + il, false); cb(Qcur_full, "Qcur_full", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); Qcur_full = ggml_reshape_4d(ctx0, Qcur_full, n_embd_head * 2, n_head, n_tokens, 1); @@ -266,12 +272,6 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn( Qcur_full->nb[1], Qcur_full->nb[2], Qcur_full->nb[3], n_embd_head * ggml_element_size(Qcur_full)); cb(gate, "gate", il); - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s); - cb(Kcur, "Kcur", il); - - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s); - cb(Vcur, "Vcur", il); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); @@ -509,6 +509,7 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); @@ -697,7 +698,11 @@ llama_model_qwen3next::graph_mtp::graph_mtp(const llama_model & model, const llm cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "mtp_attn_norm", il); - ggml_tensor * Qcur_full = build_lora_mm(layer.wq, cur, layer.wq_s); + auto [Qcur_full, Kcur, Vcur] = build_qkv(layer, cur, + n_embd_head * 2, n_head, + n_embd_head, n_head_kv, + n_embd_head, n_head_kv, + il, false); cb(Qcur_full, "mtp_Qcur_full", il); ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, @@ -708,12 +713,10 @@ llama_model_qwen3next::graph_mtp::graph_mtp(const llama_model & model, const llm Qcur = build_norm(Qcur, layer.attn_q_norm, nullptr, LLM_NORM_RMS, il); cb(Qcur, "mtp_Qcur_normed", il); - ggml_tensor * Kcur = build_lora_mm(layer.wk, cur, layer.wk_s); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il); cb(Kcur, "mtp_Kcur_normed", il); - ggml_tensor * Vcur = build_lora_mm(layer.wv, cur, layer.wv_s); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index 5596620f0782..30c08ed35e3f 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -18,6 +18,7 @@ void llama_model_qwen3vl::load_arch_tensors(llama_model_loader &) { int64_t n_vocab_out = n_vocab; if (arch == LLM_ARCH_QWEN3TTS) { + // [TAG_LLAMA_N_VOCAB_OUT] n_vocab_out = 3072; } diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 7343d4e192d9..bf09a43211aa 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -172,7 +172,8 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); // there is no output_norm: the final hyper-connection mixer carries it - hc_head_norm = create_tensor(tn(LLM_TENSOR_HC_HEAD_NORM, "weight"), { hc_dim }, 0); + // the gammas load as [n_embd, hc] so the grouped norm multiplies them without a graph reshape + hc_head_norm = create_tensor(tn(LLM_TENSOR_HC_HEAD_NORM, "weight"), { n_embd, hc }, TENSOR_ALLOW_RESHAPE); hc_head_down = create_tensor(tn(LLM_TENSOR_HC_HEAD_DOWN, "weight"), { hc_dim, hc_lr }, 0); hc_head_up = create_tensor(tn(LLM_TENSOR_HC_HEAD_UP, "weight"), { hc_lr, hc_dim }, 0); @@ -218,11 +219,11 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { const int64_t conv_dim = key_dim * 2 + value_dim; // two HC modules per layer: before the token mixer, before the MoE - layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { hc_dim }, 0); + layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE); layer.hc_attn_down = create_tensor(tn(LLM_TENSOR_HC_ATTN_DOWN, "weight", il), { hc_dim, hc_lr }, 0); layer.hc_attn_up = create_tensor(tn(LLM_TENSOR_HC_ATTN_UP, "weight", il), { hc_lr, hc_dim }, 0); layer.hc_attn_inject = create_tensor(tn(LLM_TENSOR_HC_ATTN_INJECT, "weight", il), { hc_dim, hc }, 0); - layer.hc_ffn_norm = create_tensor(tn(LLM_TENSOR_HC_FFN_NORM, "weight", il), { hc_dim }, 0); + layer.hc_ffn_norm = create_tensor(tn(LLM_TENSOR_HC_FFN_NORM, "weight", il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE); layer.hc_ffn_down = create_tensor(tn(LLM_TENSOR_HC_FFN_DOWN, "weight", il), { hc_dim, hc_lr }, 0); layer.hc_ffn_up = create_tensor(tn(LLM_TENSOR_HC_FFN_UP, "weight", il), { hc_lr, hc_dim }, 0); layer.hc_ffn_inject = create_tensor(tn(LLM_TENSOR_HC_FFN_INJECT, "weight", il), { hc_dim, hc }, 0); @@ -255,9 +256,9 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { if (hparams.is_ple(il)) { layer.ple_key = create_tensor(tn(LLM_TENSOR_PLE_KEY, "weight", il), { n_embd, hc_dim }, 0); layer.ple_value = create_tensor(tn(LLM_TENSOR_PLE_VALUE, "weight", il), { n_embd, n_embd }, 0); - layer.ple_norm_key = create_tensor(tn(LLM_TENSOR_PLE_NORM_KEY, "weight", il), { hc_dim }, 0); - layer.ple_norm_query = create_tensor(tn(LLM_TENSOR_PLE_NORM_QUERY, "weight", il), { hc_dim }, 0); - layer.ple_norm_conv = create_tensor(tn(LLM_TENSOR_PLE_NORM_CONV, "weight", il), { hc_dim }, 0); + layer.ple_norm_key = create_tensor(tn(LLM_TENSOR_PLE_NORM_KEY, "weight", il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE); + layer.ple_norm_query = create_tensor(tn(LLM_TENSOR_PLE_NORM_QUERY, "weight", il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE); + layer.ple_norm_conv = create_tensor(tn(LLM_TENSOR_PLE_NORM_CONV, "weight", il), { n_embd, hc }, TENSOR_ALLOW_RESHAPE); layer.ple_conv1d = create_tensor(tn(LLM_TENSOR_PLE_CONV1D, "weight", il), { hparams.ple_conv_kernel, hc_dim }, 0); } @@ -287,11 +288,11 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { const int64_t n_ff_exp = hparams.n_ff_exp(il) ? hparams.n_ff_exp(il) : n_ff / n_expert_used; const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff; - layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { hc_dim }, flags); + layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { n_embd, hc }, flags | TENSOR_ALLOW_RESHAPE); layer.hc_attn_down = create_tensor(tn(LLM_TENSOR_HC_ATTN_DOWN, "weight", il), { hc_dim, hc_lr }, flags); layer.hc_attn_up = create_tensor(tn(LLM_TENSOR_HC_ATTN_UP, "weight", il), { hc_lr, hc_dim }, flags); layer.hc_attn_inject = create_tensor(tn(LLM_TENSOR_HC_ATTN_INJECT, "weight", il), { hc_dim, hc }, flags); - layer.hc_ffn_norm = create_tensor(tn(LLM_TENSOR_HC_FFN_NORM, "weight", il), { hc_dim }, flags); + layer.hc_ffn_norm = create_tensor(tn(LLM_TENSOR_HC_FFN_NORM, "weight", il), { n_embd, hc }, flags | TENSOR_ALLOW_RESHAPE); layer.hc_ffn_down = create_tensor(tn(LLM_TENSOR_HC_FFN_DOWN, "weight", il), { hc_dim, hc_lr }, flags); layer.hc_ffn_up = create_tensor(tn(LLM_TENSOR_HC_FFN_UP, "weight", il), { hc_lr, hc_dim }, flags); layer.hc_ffn_inject = create_tensor(tn(LLM_TENSOR_HC_FFN_INJECT, "weight", il), { hc_dim, hc }, flags); @@ -474,55 +475,40 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix( const int64_t hc_dim = hc * n_embd; const int64_t nt = x->ne[2]; - // grouped RMSNorm: reduce over one stream, then scale all streams with the [hc_dim] gamma + // grouped RMSNorm: reduce over one stream, then scale all streams with the [n_embd, hc] gamma // the converter folded each gamma to (1 + w) - // - // WP_QWEN4EXP_FUSE_HC_NORM=1 (default off) emits the gamma multiply while the - // normed tensor is still [n_embd, hc, nt] and reshapes to [hc_dim, nt] - // afterwards, instead of reshaping first. The products are identical -- - // element (e, c, t) is multiplied by w_norm[c*n_embd + e] either way -- but - // the RMS_NORM and MUL nodes become adjacent, which is what - // ggml_cuda_can_fuse() requires (ggml-cuda.cu:5410, via ggml_can_fuse's - // strict node_idx+1 adjacency). The intervening reshape currently blocks the - // fusion on all ~97 build_hc_mix calls per decode token; the reshape is a - // view either way, so this costs nothing when the fusion does not fire. - static const bool fuse_hc_norm = [] { - const char * e = std::getenv("WP_QWEN4EXP_FUSE_HC_NORM"); - return e != nullptr && e[0] == '1'; - }(); - - ggml_tensor * xn = ggml_rms_norm(ctx0, x, hparams.f_norm_rms_eps); - if (fuse_hc_norm && ggml_is_contiguous(w_norm) && w_norm->ne[0] == hc_dim && - ggml_nrows(w_norm) == 1) { - // w_norm: [hc_dim] -> [n_embd, hc, 1], broadcast over the token axis - ggml_tensor * w3 = ggml_reshape_3d(ctx0, w_norm, n_embd, hc, 1); - xn = ggml_mul(ctx0, xn, w3); - xn = ggml_reshape_2d(ctx0, xn, hc_dim, nt); - } else { - xn = ggml_reshape_2d(ctx0, xn, hc_dim, nt); - xn = ggml_mul(ctx0, xn, w_norm); - } + ggml_tensor * xn = ggml_mul(ctx0, ggml_rms_norm(ctx0, x, hparams.f_norm_rms_eps), w_norm); + xn = ggml_reshape_2d(ctx0, xn, hc_dim, nt); cb(xn, "hc_norm", il); ggml_tensor * lo = build_lora_mm(w_down, xn); lo = ggml_silu(ctx0, ggml_scale(ctx0, lo, 1.0f / (float) hc)); - ggml_tensor * gate = ggml_sigmoid(ctx0, build_lora_mm(w_up, lo)); + ggml_tensor * gate = build_lora_mm(w_up, lo); cb(gate, "hc_gate", il); - ggml_tensor * gated = ggml_mul(ctx0, xn, gate); - gated = ggml_reshape_3d(ctx0, gated, n_embd, hc, nt); - - // collapse the streams by their mean - ggml_tensor * mixed = ggml_view_2d(ctx0, gated, n_embd, nt, - ggml_row_size(gated->type, n_embd) * hc, 0); - mixed = ggml_cont(ctx0, mixed); - for (int64_t c = 1; c < hc; ++c) { - ggml_tensor * s = ggml_view_2d(ctx0, gated, n_embd, nt, - ggml_row_size(gated->type, n_embd) * hc, - ggml_row_size(gated->type, n_embd) * c); - mixed = ggml_add(ctx0, mixed, s); + ggml_tensor * mixed = nullptr; + if (cparams.fused_dsv4_hc_pre && il >= 0) { + // sigmoid gate and mean over the streams in one op + mixed = ggml_dsv4_hc_pre_gated(ctx0, + ggml_reshape_3d(ctx0, xn, n_embd, hc, nt), + ggml_reshape_3d(ctx0, gate, n_embd, hc, nt), 1.0f / (float) hc); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_PRE, mixed, il}); + } else { + ggml_tensor * gated = ggml_mul(ctx0, xn, ggml_sigmoid(ctx0, gate)); + gated = ggml_reshape_3d(ctx0, gated, n_embd, hc, nt); + + // collapse the streams by their mean + mixed = ggml_view_2d(ctx0, gated, n_embd, nt, + ggml_row_size(gated->type, n_embd) * hc, 0); + mixed = ggml_cont(ctx0, mixed); + for (int64_t c = 1; c < hc; ++c) { + ggml_tensor * s = ggml_view_2d(ctx0, gated, n_embd, nt, + ggml_row_size(gated->type, n_embd) * hc, + ggml_row_size(gated->type, n_embd) * c); + mixed = ggml_add(ctx0, mixed, s); + } + mixed = ggml_scale(ctx0, mixed, 1.0f / (float) hc); } - mixed = ggml_scale(ctx0, mixed, 1.0f / (float) hc); cb(mixed, "hc_mixed", il); if (inject) { @@ -544,12 +530,20 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine( // 2*sigmoid centres the scatter weights on 1, so a zero injection is a plain residual add ggml_tensor * w = ggml_sigmoid(ctx0, ggml_scale(ctx0, inject, 1.0f / (float) hc)); w = ggml_scale(ctx0, w, 2.0f); - w = ggml_reshape_3d(ctx0, w, 1, hc, nt); - ggml_tensor * b = ggml_reshape_3d(ctx0, block_out, n_embd, 1, nt); - b = ggml_repeat_4d(ctx0, b, n_embd, hc, nt, 1); + ggml_tensor * cur = nullptr; + if (cparams.fused_dsv4_hc_post && il >= 0) { + // identity comb: every stream adds the same block output, scaled by its own weight + cur = ggml_dsv4_hc_post(ctx0, block_out, residual, w, nullptr); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_POST, cur, il}); + } else { + w = ggml_reshape_3d(ctx0, w, 1, hc, nt); + + ggml_tensor * b = ggml_reshape_3d(ctx0, block_out, n_embd, 1, nt); + b = ggml_repeat_4d(ctx0, b, n_embd, hc, nt, 1); - ggml_tensor * cur = ggml_add(ctx0, residual, ggml_mul(ctx0, b, w)); + cur = ggml_add(ctx0, residual, ggml_mul(ctx0, b, w)); + } cb(cur, "hc_combine", il); return cur; @@ -1221,10 +1215,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa( cb(q, "q_turbo_rot", il); } - // TODO: enable sparse attention when we are ready - // ref: https://github.com/ggml-org/llama.cpp/pull/27970 - //ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_top_k, nullptr, nullptr, top_k->ne[0], kq_scale, il); - ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_top_k, nullptr, nullptr, 0, kq_scale, il); + ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_top_k, nullptr, nullptr, top_k->ne[0], kq_scale, il); cb(cur, "kqv_out", il); // the rotation is its own inverse, so undo it on the value side of the output @@ -1414,6 +1405,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); @@ -1686,13 +1678,10 @@ ggml_tensor * llama_model_qwen4exp::graph::build_ple( ggml_tensor * key = build_lora_mm(model.layers[il].ple_key, emb); ggml_tensor * value = build_lora_mm(model.layers[il].ple_value, emb); - // both norms group over one hc stream, with a weight over the whole hc*n_embd layout + // both norms group over one hc stream, with a [n_embd, hc] weight auto grouped_norm = [&](ggml_tensor * x, ggml_tensor * w) { ggml_tensor * t = ggml_reshape_3d(ctx0, x, n_embd, hc, n_tokens); - t = ggml_rms_norm(ctx0, t, hparams.f_norm_rms_eps); - t = ggml_reshape_2d(ctx0, t, hc_dim, n_tokens); - t = ggml_mul(ctx0, t, w); - return ggml_reshape_3d(ctx0, t, n_embd, hc, n_tokens); + return ggml_mul(ctx0, ggml_rms_norm(ctx0, t, hparams.f_norm_rms_eps), w); }; key = grouped_norm(key, model.layers[il].ple_norm_key); diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cpp index 680ffb8fda37..555f8b718470 100644 --- a/src/models/smallthinker.cpp +++ b/src/models/smallthinker.cpp @@ -6,9 +6,7 @@ void llama_model_smallthinker::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, true); + load_swa_pattern(ml, 4, true); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/spark2-5.cpp b/src/models/spark2-5.cpp new file mode 100644 index 000000000000..107448777c5c --- /dev/null +++ b/src/models/spark2-5.cpp @@ -0,0 +1,146 @@ +#include "models.h" + +void llama_model_spark2_5::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); + + switch (hparams.n_layer()) { + case 28: type = LLM_TYPE_1_7B; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_spark2_5::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == nullptr) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + const int64_t n_head_i = hparams.n_head(i); + const int64_t n_head_kv_i = hparams.n_head_kv(i); + const int64_t n_embd_q = hparams.n_embd_head_k(i) * n_head_i; + const int64_t n_embd_k = hparams.n_embd_head_k(i) * n_head_kv_i; + const int64_t n_embd_v = hparams.n_embd_head_v(i) * n_head_kv_i; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + create_tensor_qkv(layer, i, n_embd, n_embd_q, n_embd_k, n_embd_v, 0); + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head_i}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + } +} + +std::unique_ptr<llm_graph_context> llama_model_spark2_5::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique<graph>(*this, params); +} + +llama_model_spark2_5::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + GGML_ASSERT(hparams.swa_type == LLAMA_SWA_TYPE_STANDARD); + + ggml_tensor * inpL = build_inp_embd(model.tok_embd); + ggml_tensor * inp_pos = build_inp_pos(); + auto * inp_attn = build_attn_inp_kv_iswa(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + ggml_tensor * cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + const int64_t n_head_i = hparams.n_head(il); + const int64_t n_head_kv_i = hparams.n_head_kv(il); + const int64_t n_rot_i = hparams.n_rot(il); + const float freq_base_i = model.get_rope_freq_base(cparams, il); + const float freq_scale_i = model.get_rope_freq_scale(cparams, il); + + ggml_tensor * attn_inp = cur; + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head_i, n_head_kv_i, il); + + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, + n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i, + ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, + n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur_rope", il); + cb(Kcur, "Kcur_rope", il); + + cur = build_attn(inp_attn, + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(cur, "attn_out", il); + + ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, attn_inp); + gate = ggml_sigmoid(ctx0, gate); + cb(gate, "attn_gate", il); + + const int64_t n_tokens_i = cur->ne[1]; + cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head_i, n_tokens_i); + gate = ggml_reshape_3d(ctx0, gate, 1, n_head_i, n_tokens_i); + cur = ggml_mul(ctx0, cur, gate); + cur = ggml_reshape_2d(ctx0, cur, n_embd_head * n_head_i, n_tokens_i); + cb(cur, "attn_gated", il); + + cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s); + cb(cur, "attn_out_proj", il); + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, nullptr, nullptr, + model.layers[il].ffn_gate, nullptr, nullptr, + model.layers[il].ffn_down, nullptr, nullptr, + nullptr, + LLM_FFN_GELU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + ggml_tensor * cur = build_norm(inpL, model.output_norm, nullptr, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/step35.cpp b/src/models/step35.cpp index 53f3179c6357..ca68855d8dab 100644 --- a/src/models/step35.cpp +++ b/src/models/step35.cpp @@ -23,7 +23,7 @@ void llama_model_step35::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer_all); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all, false); ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, false); @@ -216,9 +216,11 @@ llama_model_step35::graph::graph(const llama_model & model, const llm_graph_para { cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); - ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head_k, n_head_l, + n_embd_head_k, n_head_kv_l, + n_embd_head_v, n_head_kv_l, + il, false); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -425,9 +427,11 @@ llama_model_step35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "mtp_attn_norm", il); - ggml_tensor * Qcur = build_lora_mm(layer.wq, cur, layer.wq_s); - ggml_tensor * Kcur = build_lora_mm(layer.wk, cur, layer.wk_s); - ggml_tensor * Vcur = build_lora_mm(layer.wv, cur, layer.wv_s); + auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, + n_embd_head_k, n_head_l, + n_embd_head_k, n_head_kv_l, + n_embd_head_v, n_head_kv_l, + il, false); cb(Qcur, "mtp_Qcur", il); cb(Kcur, "mtp_Kcur", il); cb(Vcur, "mtp_Vcur", il); diff --git a/src/models/t5.cpp b/src/models/t5.cpp index b0e3f062572f..e2bf12b6b151 100644 --- a/src/models/t5.cpp +++ b/src/models/t5.cpp @@ -106,18 +106,6 @@ void llama_model_t5::load_arch_tensors(llama_model_loader &) { } } -std::unique_ptr<llm_graph_context> llama_model_t5::build_arch_graph(const llm_graph_params & params) const { - switch (params.gtype) { - case LLM_GRAPH_TYPE_ENCODER: - return std::make_unique<graph<true>>(*this, params); - case LLM_GRAPH_TYPE_DEFAULT: - case LLM_GRAPH_TYPE_DECODER: - return std::make_unique<graph<false>>(*this, params); - default: - GGML_ABORT("invalid graph type"); - }; -} - template <> llama_model_t5::graph<false>::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -368,3 +356,15 @@ llama_model_t5::graph<true>::graph(const llama_model & model, const llm_graph_pa ggml_build_forward_expand(gf, cur); } + +std::unique_ptr<llm_graph_context> llama_model_t5::build_arch_graph(const llm_graph_params & params) const { + switch (params.gtype) { + case LLM_GRAPH_TYPE_ENCODER: + return std::make_unique<graph<true>>(*this, params); + case LLM_GRAPH_TYPE_DEFAULT: + case LLM_GRAPH_TYPE_DECODER: + return std::make_unique<graph<false>>(*this, params); + default: + GGML_ABORT("invalid graph type"); + }; +} diff --git a/tests/.gitignore b/tests/.gitignore index 9e076a3e140e..982d3abffbf0 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -4,6 +4,7 @@ !stress/ !perf-baseline/ !perf-baseline/** +!fusion/ *.o ggml-common.h **/*.swp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8b7659cac503..9f2314f72ccb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -10,7 +10,7 @@ function(llama_build source) endif() add_executable(${TEST_TARGET} ${TEST_SOURCES}) - target_link_libraries(${TEST_TARGET} PRIVATE llama llama-common) + target_link_libraries(${TEST_TARGET} PRIVATE llama llama-common) # TODO: [TAG_TESTS_LLAMA_LINK] if (LLAMA_TESTS_INSTALL) install(TARGETS ${TEST_TARGET} RUNTIME) endif() @@ -166,11 +166,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) llama_build_and_test(test-server-models-estimate-key.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) target_include_directories(test-server-models-estimate-key PRIVATE ${PROJECT_SOURCE_DIR}/tools/server) target_link_libraries(test-server-models-estimate-key PRIVATE llama-server-impl) - # TODO: disabled on loongarch64 because the ggml-ci node lacks Python 3.8 - if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64") - llama_build_and_test(test-json-schema-to-grammar.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) - target_include_directories(test-json-schema-to-grammar PRIVATE ${PROJECT_SOURCE_DIR}/tools/server) - endif() + llama_build_and_test(test-json-schema-to-grammar.cpp) if (NOT GGML_BACKEND_DL) llama_build(test-quantize-stats.cpp) @@ -199,7 +195,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) # llama_build_and_test(test-double-float.cpp) # SLOW - llama_build_and_test(test-llama-archs.cpp) + llama_build(test-llama-archs.cpp) llama_build_and_test(test-dsv41-load.cpp) llama_build_and_test(test-dsv41-decode.cpp) llama_build_and_test(test-dsv41-spine-load.cpp) @@ -245,6 +241,15 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) set_tests_properties(test-recurrent-state-rollback-dsv4 PROPERTIES FIXTURES_REQUIRED generate-models ) + llama_test( + test-recurrent-state-rollback + NAME test-recurrent-state-rollback-kimi-k3 + LABEL main + ARGS -m "${MODEL_DIR}/kimi-k3-moe.gguf" + ) + set_tests_properties(test-recurrent-state-rollback-kimi-k3 PROPERTIES + FIXTURES_REQUIRED generate-models + ) # Test state save/load functionality across all architectures, using the generated dummy models llama_test( @@ -253,11 +258,14 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) ARGS --models "${MODEL_DIR}" ) set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED generate-models) + + llama_build(test-fusion.cpp) endif() llama_build_and_test(test-chat-peg-parser.cpp peg-parser/simple-tokenize.cpp) llama_build_and_test(test-jinja.cpp) llama_test(test-jinja NAME test-jinja-py ARGS -py LABEL python) +llama_build_and_test(test-json-schema.cpp) llama_build_and_test(test-chat-auto-parser.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) llama_build_and_test(test-chat-template.cpp) # debug tool for chat template differential analysis (not registered as a test, run it manually) @@ -288,7 +296,7 @@ add_test(NAME test-download-model COMMAND ${CMAKE_COMMAND} -DDEST=${MODEL_DEST} -DNAME=${MODEL_NAME} -DHASH=${MODEL_HASH} - -P ${CMAKE_SOURCE_DIR}/cmake/download-models.cmake + -P ${PROJECT_SOURCE_DIR}/cmake/download-models.cmake ) set_tests_properties(test-download-model PROPERTIES FIXTURES_SETUP test-download-model) @@ -305,7 +313,9 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC) # TODO: repair known memory leaks llama_build_and_test(test-opt.cpp) endif() -llama_build_and_test(test-backend-ops.cpp) + +# TODO: make this test (and others) not link `libllama` as it is not needed [TAG_TESTS_LLAMA_LINK] +llama_build(test-backend-ops.cpp) llama_build_and_test(test-wp-mul-mat-pin.cpp) llama_build_and_test(test-paged-attn-vk.cpp) @@ -668,14 +678,6 @@ target_include_directories(test-wp-prefetch-hints PRIVATE target_compile_features(test-wp-prefetch-hints PRIVATE cxx_std_17) llama_test(test-wp-prefetch-hints) -# the tensor API kernels come from a separate metallib - check they produce correct results -# ref: https://github.com/ggml-org/llama.cpp/issues/27473 -if (GGML_METAL AND NOT GGML_METAL_EMBED_LIBRARY) - llama_test(test-backend-ops NAME test-backend-ops-metallib-tensor - ARGS test -b MTL0 -o MUL_MAT -p type_a=q6_K) - set_tests_properties(test-backend-ops-metallib-tensor PROPERTIES ENVIRONMENT GGML_METAL_TENSOR_ENABLE=1) -endif() - llama_build_and_test(test-model-load-cancel.cpp LABEL "model") llama_build_and_test(test-autorelease.cpp LABEL "model") llama_build_and_test(test-backend-sampler.cpp LABEL "model") @@ -689,7 +691,7 @@ if (APPLE) llama_build(test-rset-release.cpp) endif() -if (NOT GGML_BACKEND_DL) +if (NOT GGML_BACKEND_DL AND GGML_CPU) # these tests use the backends directly and cannot be built with dynamic loading llama_build_and_test(test-barrier.cpp) llama_build_and_test(test-quantize-fns.cpp) diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv new file mode 100644 index 000000000000..3970da970014 --- /dev/null +++ b/tests/fusion/MTL.csv @@ -0,0 +1,266 @@ +# test-fusion baseline for device MTL +# arch ,moe ,mode ,label , count +afmoe ,1 ,any ,MUL+ADD , 2 +afmoe ,1 ,any ,RMS_NORM+MUL , 10 +afmoe ,1 ,any ,RMS_NORM+MUL+ADD , 3 +arcee ,0 ,any ,RMS_NORM+MUL , 5 +arctic ,0 ,any ,MUL+ADD , 4 +arctic ,0 ,any ,RMS_NORM+MUL , 7 +arctic ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +baichuan ,0 ,any ,RMS_NORM+MUL , 5 +bailingmoe ,1 ,any ,ADD+ADD , 2 +bailingmoe ,1 ,any ,MUL+ADD , 4 +bailingmoe ,1 ,any ,RMS_NORM+MUL , 5 +bailingmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2 +bailingmoe2 ,1 ,any ,ADD+ADD , 1 +bailingmoe2 ,1 ,any ,MUL+ADD , 2 +bailingmoe2 ,1 ,any ,RMS_NORM+MUL , 9 +bailingmoe3 ,1 ,any ,ADD+ADD , 1 +bailingmoe3 ,1 ,any ,GATED_DELTA_NET+CPY , 1 +bailingmoe3 ,1 ,any ,MUL+ADD , 2 +bailingmoe3 ,1 ,any ,RMS_NORM+MUL , 8 +bailingmoe3 ,1 ,any ,RMS_NORM+SCALE , 2 +bloom ,0 ,any ,NORM+MUL+ADD , 6 +chatglm ,0 ,any ,RMS_NORM+MUL , 5 +codeshell ,0 ,any ,NORM+MUL+ADD , 5 +cogvlm ,0 ,any ,RMS_NORM+MUL , 5 +cohere2 ,0 ,any ,ADD+ADD , 2 +cohere2 ,0 ,any ,NORM+MUL , 3 +cohere2moe ,1 ,any ,ADD+ADD , 2 +cohere2moe ,1 ,any ,MUL+ADD , 2 +cohere2moe ,1 ,any ,RMS_NORM+MUL , 3 +command-r ,0 ,any ,NORM+MUL , 3 +dbrx ,0 ,any ,MUL+ADD , 4 +dbrx ,0 ,any ,NORM+MUL , 5 +dbrx ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +deci ,0 ,any ,RMS_NORM+MUL , 5 +deepseek ,0 ,any ,ADD+ADD , 1 +deepseek ,0 ,any ,MUL+ADD , 2 +deepseek ,0 ,any ,RMS_NORM+MUL , 5 +deepseek ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 1 +deepseek2 ,0 ,any ,ADD+ADD , 1 +deepseek2 ,0 ,any ,MUL+ADD , 2 +deepseek2 ,0 ,any ,RMS_NORM+MUL , 9 +deepseek32 ,0 ,any ,ADD+ADD , 1 +deepseek32 ,0 ,any ,MUL+ADD , 2 +deepseek32 ,0 ,any ,NORM+MUL+ADD , 2 +deepseek32 ,0 ,any ,RMS_NORM+MUL , 9 +deepseek4 ,0 ,any ,MUL+ADD , 8 +deepseek4 ,0 ,any ,RMS_NORM+MUL , 20 +dots1 ,0 ,any ,ADD+ADD , 1 +dots1 ,0 ,any ,MUL+ADD , 2 +dots1 ,0 ,any ,RMS_NORM+MUL , 9 +dots3note ,0 ,any ,ADD+ADD , 1 +dots3note ,0 ,any ,MUL+ADD , 2 +dots3note ,0 ,any ,NORM+MUL+ADD , 1 +dots3note ,0 ,any ,RMS_NORM+MUL , 11 +dream ,0 ,any ,RMS_NORM+MUL , 5 +ernie4_5-moe ,1 ,any ,ADD+ADD , 1 +ernie4_5-moe ,1 ,any ,MUL+ADD , 2 +ernie4_5-moe ,1 ,any ,RMS_NORM+MUL , 5 +ernie4_5 ,0 ,any ,RMS_NORM+MUL , 5 +exaone ,0 ,any ,RMS_NORM+MUL , 5 +exaone-moe ,1 ,any ,ADD+ADD , 1 +exaone-moe ,1 ,any ,MUL+ADD , 2 +exaone-moe ,1 ,any ,RMS_NORM+MUL , 9 +exaone4 ,0 ,any ,RMS_NORM+MUL , 5 +exaone4 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +falcon ,0 ,any ,ADD+ADD , 2 +falcon ,0 ,any ,NORM+MUL+ADD , 5 +falcon-h1 ,0 ,any ,ADD+ADD , 2 +falcon-h1 ,0 ,any ,RMS_NORM+MUL , 9 +gemma ,0 ,any ,RMS_NORM+MUL , 5 +gemma2 ,0 ,any ,RMS_NORM+MUL , 5 +gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +gemma3 ,0 ,any ,RMS_NORM+MUL , 9 +gemma3 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +glm-dsa ,0 ,any ,ADD+ADD , 1 +glm-dsa ,0 ,any ,MUL+ADD , 2 +glm-dsa ,0 ,any ,NORM+MUL+ADD , 2 +glm-dsa ,0 ,any ,RMS_NORM+MUL , 9 +glm4 ,0 ,any ,RMS_NORM+MUL , 5 +glm4 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +glm4moe ,1 ,any ,ADD+ADD , 1 +glm4moe ,1 ,any ,MUL+ADD , 2 +glm4moe ,1 ,any ,RMS_NORM+MUL , 9 +gpt-oss ,0 ,any ,MUL+ADD , 4 +gpt-oss ,0 ,any ,RMS_NORM+MUL , 5 +gpt2 ,0 ,any ,NORM+MUL+ADD , 5 +gptneox ,0 ,any ,NORM+MUL+ADD , 5 +granite ,0 ,any ,RMS_NORM+MUL , 5 +granite ,0 ,any ,MUL+ADD , 4 +granite ,0 ,any ,RMS_NORM+MUL , 5 +granite ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +granite_swa ,0 ,any ,RMS_NORM+MUL , 5 +granitehybrid ,0 ,any ,RMS_NORM+MUL , 6 +granitemoe ,1 ,any ,RMS_NORM+MUL , 5 +granitemoe ,1 ,any ,MUL+ADD , 4 +granitemoe ,1 ,any ,RMS_NORM+MUL , 5 +granitemoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +grok ,0 ,any ,MUL+ADD , 4 +grok ,0 ,any ,RMS_NORM+MUL , 5 +grok ,0 ,any ,RMS_NORM+MUL+ADD , 4 +grok ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +grovemoe ,1 ,any ,ADD+ADD , 2 +grovemoe ,1 ,any ,MUL+ADD , 8 +grovemoe ,1 ,any ,RMS_NORM+MUL , 9 +hunyuan-dense ,0 ,any ,RMS_NORM+MUL , 9 +hunyuan-moe ,1 ,any ,ADD+ADD , 2 +hunyuan-moe ,1 ,any ,MUL+ADD , 4 +hunyuan-moe ,1 ,any ,RMS_NORM+MUL , 9 +hunyuan-moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +hunyuan_vl ,0 ,any ,RMS_NORM+MUL , 9 +hy_v3 ,0 ,any ,ADD+ADD , 2 +hy_v3 ,0 ,any ,MUL+ADD , 4 +hy_v3 ,0 ,any ,RMS_NORM+MUL , 9 +hy_v4 ,0 ,any ,MUL+ADD , 2 +hy_v4 ,0 ,any ,NORM+MUL+ADD , 1 +hy_v4 ,0 ,any ,RMS_NORM+MUL , 9 +internlm2 ,0 ,any ,RMS_NORM+MUL , 5 +jais ,0 ,any ,NORM+MUL+ADD , 5 +jais2 ,0 ,any ,NORM+MUL+ADD , 5 +jamba ,0 ,any ,RMS_NORM+MUL , 8 +kimi-k3 ,0 ,any ,GATED_DELTA_NET+CPY , 1 +kimi-k3 ,0 ,any ,MUL+ADD , 2 +kimi-k3 ,0 ,any ,RMS_NORM+MUL , 17 +kimi-k3 ,0 ,any ,RMS_NORM+SCALE , 2 +kimi-linear ,0 ,any ,ADD+ADD , 1 +kimi-linear ,0 ,any ,GATED_DELTA_NET+CPY , 1 +kimi-linear ,0 ,any ,MUL+ADD , 2 +kimi-linear ,0 ,any ,RMS_NORM+MUL , 7 +kimi-linear ,0 ,any ,RMS_NORM+SCALE , 2 +laguna ,0 ,any ,ADD+ADD , 1 +laguna ,0 ,any ,MUL+ADD , 2 +laguna ,0 ,any ,RMS_NORM+MUL , 9 +lfm2 ,0 ,any ,RMS_NORM+MUL , 7 +lfm2moe ,1 ,any ,MUL+ADD , 2 +lfm2moe ,1 ,any ,RMS_NORM+MUL , 7 +llada ,0 ,any ,RMS_NORM+MUL , 5 +llada-moe ,1 ,any ,MUL+ADD , 4 +llada-moe ,1 ,any ,RMS_NORM+MUL , 9 +llada-moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2 +llama ,0 ,any ,RMS_NORM+MUL , 5 +llama ,0 ,any ,MUL+ADD , 4 +llama ,0 ,any ,RMS_NORM+MUL , 5 +llama ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +llama4 ,0 ,any ,ADD+ADD , 2 +llama4 ,0 ,any ,RMS_NORM+MUL , 9 +maincoder ,0 ,any ,RMS_NORM+MUL , 9 +mamba ,0 ,any ,RMS_NORM+MUL , 3 +mamba2 ,0 ,any ,RMS_NORM+MUL , 5 +maple ,0 ,any ,MUL+ADD , 4 +maple ,0 ,any ,RMS_NORM+MUL , 9 +maple ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +mellum ,0 ,any ,MUL+ADD , 4 +mellum ,0 ,any ,RMS_NORM+MUL , 9 +mellum ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +mimo2 ,0 ,any ,MUL+ADD , 4 +mimo2 ,0 ,any ,RMS_NORM+MUL , 5 +minicpm ,0 ,any ,RMS_NORM+MUL , 5 +minicpm ,0 ,any ,MUL+ADD , 4 +minicpm ,0 ,any ,RMS_NORM+MUL , 5 +minicpm ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +minicpm3 ,0 ,any ,RMS_NORM+MUL , 9 +minimax-01 ,0 ,any ,MUL+ADD , 4 +minimax-01 ,0 ,any ,RMS_NORM+MUL , 6 +minimax-01 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +minimax-m2 ,0 ,any ,MUL+ADD , 4 +minimax-m2 ,0 ,any ,RMS_NORM+MUL , 9 +minimax-m3 ,0 ,any ,ADD+ADD , 1 +minimax-m3 ,0 ,any ,MUL+ADD , 2 +minimax-m3 ,0 ,any ,RMS_NORM+MUL , 11 +mistral3 ,0 ,any ,RMS_NORM+MUL , 5 +mistral3 ,0 ,any ,MUL+ADD , 4 +mistral3 ,0 ,any ,RMS_NORM+MUL , 5 +mistral3 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +mistral4 ,0 ,any ,ADD+ADD , 1 +mistral4 ,0 ,any ,MUL+ADD , 2 +mistral4 ,0 ,any ,RMS_NORM+MUL , 9 +mpt ,0 ,any ,NORM+MUL+ADD , 5 +muse-glimmer ,0 ,any ,RMS_NORM+MUL , 10 +muse-glimmer ,0 ,any ,RMS_NORM+MUL+ADD , 3 +nanbeige ,0 ,any ,RMS_NORM+MUL , 5 +nemotron ,0 ,any ,NORM+MUL+ADD , 5 +nemotron_h ,0 ,any ,RMS_NORM+MUL , 5 +nemotron_h_moe ,1 ,any ,RMS_NORM+MUL , 5 +olmo2 ,0 ,any ,RMS_NORM+MUL , 5 +olmo2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +olmoe ,1 ,any ,MUL+ADD , 4 +olmoe ,1 ,any ,RMS_NORM+MUL , 9 +olmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2 +openelm ,0 ,any ,RMS_NORM+MUL , 9 +orion ,0 ,any ,NORM+MUL+ADD , 5 +paddleocr ,0 ,any ,RMS_NORM+MUL , 5 +pangu-embedded ,0 ,any ,RMS_NORM+MUL , 5 +phi2 ,0 ,any ,ADD+ADD , 2 +phi2 ,0 ,any ,NORM+MUL+ADD , 3 +phi3 ,0 ,any ,RMS_NORM+MUL , 5 +phimoe ,1 ,any ,MUL+ADD , 4 +phimoe ,1 ,any ,RMS_NORM+MUL+ADD , 5 +phimoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +plamo ,0 ,any ,ADD+ADD , 2 +plamo ,0 ,any ,RMS_NORM+MUL , 3 +plamo2 ,0 ,any ,RMS_NORM+MUL , 10 +plamo2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +plamo2 ,0 ,any ,SSM_CONV+UNARY , 1 +plamo3 ,0 ,any ,RMS_NORM+MUL , 9 +plamo3 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +pockettts ,0 ,any ,NORM+MUL+ADD , 5 +qwen ,0 ,any ,RMS_NORM+MUL , 5 +qwen2 ,0 ,any ,RMS_NORM+MUL , 5 +qwen2moe ,1 ,any ,ADD+ADD , 2 +qwen2moe ,1 ,any ,MUL+ADD , 4 +qwen2moe ,1 ,any ,RMS_NORM+MUL , 5 +qwen2moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2 +qwen2vl ,0 ,any ,RMS_NORM+MUL , 5 +qwen3 ,0 ,any ,RMS_NORM+MUL , 9 +qwen35 ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen35 ,0 ,any ,RMS_NORM+MUL , 8 +qwen35 ,0 ,any ,RMS_NORM+SCALE , 2 +qwen35 ,0 ,any ,SSM_CONV+UNARY , 1 +qwen35moe ,1 ,any ,ADD+ADD , 2 +qwen35moe ,1 ,any ,GATED_DELTA_NET+CPY , 1 +qwen35moe ,1 ,any ,MUL+ADD , 4 +qwen35moe ,1 ,any ,RMS_NORM+MUL , 8 +qwen35moe ,1 ,any ,RMS_NORM+SCALE , 2 +qwen35moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +qwen35moe ,1 ,any ,SSM_CONV+UNARY , 1 +qwen3moe ,1 ,any ,MUL+ADD , 4 +qwen3moe ,1 ,any ,RMS_NORM+MUL , 9 +qwen3moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +qwen3next ,0 ,any ,ADD+ADD , 2 +qwen3next ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen3next ,0 ,any ,MUL+ADD , 4 +qwen3next ,0 ,any ,RMS_NORM+MUL , 8 +qwen3next ,0 ,any ,RMS_NORM+SCALE , 2 +qwen3next ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +qwen3next ,0 ,any ,SSM_CONV+UNARY , 1 +qwen3tts ,0 ,any ,RMS_NORM+MUL , 9 +qwen3vl ,0 ,any ,RMS_NORM+MUL , 9 +qwen3vlmoe ,1 ,any ,MUL+ADD , 4 +qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9 +qwen3vlmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +qwen4exp ,0 ,any ,ADD+ADD+ADD , 1 +qwen4exp ,0 ,any ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9 +qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen4exp ,0 ,any ,MUL+ADD , 4 +qwen4exp ,0 ,any ,RMS_NORM+MUL , 13 +qwen4exp ,0 ,any ,RMS_NORM+SCALE , 2 +qwen4exp ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +qwen4exp ,0 ,any ,SSM_CONV+UNARY , 1 +refact ,0 ,any ,RMS_NORM+MUL , 5 +refact ,0 ,any ,RMS_NORM+MUL , 5 +rnd1 ,0 ,any ,MUL+ADD , 4 +rnd1 ,0 ,any ,RMS_NORM+MUL , 9 +rnd1 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +seed_oss ,0 ,any ,RMS_NORM+MUL , 5 +smallthinker ,0 ,any ,MUL+ADD , 4 +smallthinker ,0 ,any ,RMS_NORM+MUL , 5 +smollm3 ,0 ,any ,RMS_NORM+MUL , 5 +spark2_5 ,0 ,any ,RMS_NORM+MUL , 5 +stablelm ,0 ,any ,NORM+MUL , 4 +stablelm ,0 ,any ,NORM+MUL+ADD , 5 +starcoder ,0 ,any ,NORM+MUL+ADD , 5 +starcoder2 ,0 ,any ,NORM+MUL+ADD , 5 +talkie ,0 ,any ,ADD+ADD , 2 +xverse ,0 ,any ,RMS_NORM+MUL , 5 diff --git a/tests/fusion/README.md b/tests/fusion/README.md new file mode 100644 index 000000000000..3ac02b061e9a --- /dev/null +++ b/tests/fusion/README.md @@ -0,0 +1,26 @@ +# Fusion baselines + +Per-device baselines for `test-fusion`, one CSV per backend (e.g. `MTL.csv`). Rows are +`arch,moe,mode,label,count`. Regenerate a CSV whenever fusion patterns change. + +## Update a baseline + +```sh +cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_METAL=ON # enable the target backend +cmake --build build --config Release --target test-llama-archs --target test-fusion -j + +rm -rf build-ci-models && mkdir -p build-ci-models +./build/bin/test-llama-archs -o build-ci-models + +./build/bin/test-fusion --models build-ci-models --device MTL0 --record MTL.csv +``` + +## Validate + +```sh +./build/bin/test-fusion --models build-ci-models --device MTL0 --check MTL.csv +``` + +Non-zero exit means a row differs from the baseline. Use `--model FILE` to run a single +architecture. Note `--check` only sees present rows — a fusion that stops matching is not +reported, so diff the recorded CSV to catch removed patterns. diff --git a/tests/peg-parser/test-unicode.cpp b/tests/peg-parser/test-unicode.cpp index 24663d7017d4..2eaafa1742cd 100644 --- a/tests/peg-parser/test-unicode.cpp +++ b/tests/peg-parser/test-unicode.cpp @@ -273,19 +273,35 @@ void test_unicode(testing &t) { }); t.test("malformed UTF-8", [](testing &t) { - std::vector<test_case> test_cases { + struct passthrough_case { + std::string input; + std::string expected_text; + std::string expected_sanitized; + }; + + std::vector<passthrough_case> test_cases { // Invalid UTF-8 bytes - {std::string("Hello\xFF\xFE"), "", COMMON_PEG_PARSE_RESULT_FAIL}, + {std::string("Hello\xFF\xFE</tag>"), std::string("Hello\xFF\xFE"), "Hello\xEF\xBF\xBD\xEF\xBF\xBD"}, // Continuation byte without lead byte - {std::string("Hello\x80World"), "", COMMON_PEG_PARSE_RESULT_FAIL}, + {std::string("Hello\x80World</tag>"), std::string("Hello\x80World"), "Hello\xEF\xBF\xBDWorld"}, - // Invalid continuation byte - {std::string("\xC3\x28"), "", COMMON_PEG_PARSE_RESULT_FAIL}, + // Invalid continuation byte, the lead byte is dropped and '(' survives + {std::string("\xC3\x28</tag>"), std::string("\xC3\x28"), "\xEF\xBF\xBD("}, + + // Two good bytes of a 3-byte sequence then a bad third byte, the prefix is replaced once and the third byte is kept + {std::string("\xE4\xB8" "A</tag>"), std::string("\xE4\xB8" "A"), "\xEF\xBF\xBD" "A"}, + {std::string("\xE4\xB8</tag>"), std::string("\xE4\xB8"), "\xEF\xBF\xBD"}, + + // Truncated sequence in a complete input, the leftover prefix is replaced once + {std::string("Hello\xE4\xB8"), std::string("Hello\xE4\xB8"), "Hello\xEF\xBF\xBD"}, + + // Valid multi-byte content around the bad byte is left alone + {std::string("\xE4\xBD\xA0\xFF\xE5\xA5\xBD</tag>"), std::string("\xE4\xBD\xA0\xFF\xE5\xA5\xBD"), "\xE4\xBD\xA0\xEF\xBF\xBD\xE5\xA5\xBD"}, }; auto parser = build_peg_parser([](common_peg_parser_builder& p) { - return p.until("</tag>"); + return p.tag("body", p.until("</tag>")) + p.optional(p.literal("</tag>")); }); for (size_t i = 0; i < test_cases.size(); i++) { @@ -296,10 +312,28 @@ void test_unicode(testing &t) { common_peg_parse_context ctx(tc.input); auto result = parser.parse(ctx); - assert_result_equal(t, tc.expected_result, result.type); + assert_result_equal(t, COMMON_PEG_PARSE_RESULT_SUCCESS, result.type); + const auto & node = ctx.ast.get(result.nodes[0]); + t.assert_equal("raw text", tc.expected_text, std::string(node.text)); + t.assert_equal("sanitized text", tc.expected_sanitized, node.sanitized_text()); }); } }); + + t.test("malformed UTF-8 rescanned by backtracking", [](testing &t) { + // The failed alternative and the lookahead scan the same bad byte, it must only be recorded once + auto parser = build_peg_parser([](common_peg_parser_builder& p) { + return (p.until("<a>") + p.literal("<a>")) | (p.peek(p.until("<b>")) + p.until("<b>") + p.literal("<b>")); + }); + + std::string input("x\xFFy<b>"); + common_peg_parse_context ctx(input); + auto result = parser.parse(ctx); + + assert_result_equal(t, COMMON_PEG_PARSE_RESULT_SUCCESS, result.type); + t.assert_equal("invalid count", 1u, result.invalid_utf8.size()); + t.assert_equal("invalid offset", 1u, result.invalid_utf8[0].pos); + }); }); t.test("json_string parser", [](testing &t) { diff --git a/tests/snapshots/nemotron-nano-3-30b-a3b.schema b/tests/snapshots/nvidia-nemotron-3-nano-30b-a3b.schema similarity index 100% rename from tests/snapshots/nemotron-nano-3-30b-a3b.schema rename to tests/snapshots/nvidia-nemotron-3-nano-30b-a3b.schema diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 8d6e923e38d8..624f7d3120c3 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -57,7 +57,7 @@ static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float m std::vector<float> data(nels); { // parallel initialization - static const size_t n_threads = N_THREADS; + static const size_t n_threads = std::max<size_t>(1, std::min<size_t>(nels/1024, std::min<size_t>(4, N_THREADS/2))); auto init_thread = [&](size_t start, size_t end) { thread_local std::default_random_engine gen(std::random_device{}()); @@ -463,17 +463,9 @@ static std::string var_to_str(ggml_scale_mode mode) { #define VARS_TO_STR16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) VAR_TO_STR(a) + "," + VARS_TO_STR15(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) #define VARS_TO_STR17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) VAR_TO_STR(a) + "," + VARS_TO_STR16(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) -#ifdef GGML_USE_SYCL -static bool inline _isinf(float f) { - return (*(uint32_t *)&f & 0x7fffffff) == 0x7f800000; -} -#else -static bool inline _isinf(float f) { return std::isinf(f); } -#endif - // accept FLT_MAX as infinity static bool isinf_or_max(float f) { - return _isinf(f) || f == FLT_MAX || f == -FLT_MAX; + return std::isinf(f) || f == FLT_MAX || f == -FLT_MAX; } static bool ggml_is_view_op(enum ggml_op op) { @@ -1154,6 +1146,47 @@ static void print_test_result_locked(printer * output_printer, const test_result output_printer->print_test_result(result); } +// Splits the -o filter into comma separated entries. Commas inside parentheses +// (i.e. inside a full test case string) are not treated as separators. +static std::vector<std::string_view> op_filter_entries(const char * op_names_filter) { + std::vector<std::string_view> entries; + if (op_names_filter == nullptr) { + return entries; + } + std::string_view filter(op_names_filter); + while (!filter.empty()) { + auto comma_pos = filter.find_first_of(','); + const auto lparen_pos = filter.find_first_of('('); + if (lparen_pos < comma_pos) { + const auto rparen_pos = filter.find_first_of(')'); + comma_pos = filter.find_first_of(',', rparen_pos); + } + entries.push_back(filter.substr(0, comma_pos)); + filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : ""; + } + return entries; +} + +// An entry from the -o filter matches an op if it is either +// * an exact op name as given by ggml_op_desc() (e.g. "ADD"), or +// * a regex that matches the op name (e.g. "DSV4.*") +static bool op_filter_entry_matches(std::string_view entry, std::string_view op_name) { + if (entry == op_name) { + return true; + } + // plain op names are matched exactly, anything else is treated as a regex + if (std::regex_match(std::string(entry), std::regex("[A-Z0-9_]+"))) { + return false; + } + std::regex re; + try { + re = std::regex(std::string(entry)); + } catch (const std::regex_error &) { + return false; + } + return std::regex_search(op_name.data(), op_name.data() + op_name.size(), re); +} + struct test_case { virtual ~test_case() {} @@ -1228,6 +1261,11 @@ struct test_case { } } + // re-draw data-dependent inputs between timed perf iterations + virtual void reinit_perf_iter(ggml_context * ctx) { + GGML_UNUSED(ctx); + } + virtual size_t op_size(ggml_tensor * t) { size_t size = ggml_nbytes(t); // add source tensors @@ -1312,34 +1350,24 @@ struct test_case { return t; } - // Checks an op against the test filter, which is a comma separated list of OP names or specific variations + // Checks an op against the test filter, which is a comma separated list of OP names, regexes, or specific variations bool matches_filter(ggml_tensor * op, const char * op_names_filter) { - if (op_names_filter) { - const auto op_name = op_desc(op); - const auto op_full_name = op_name + "(" + vars() + ")"; - std::string_view filter(op_names_filter); - while (!filter.empty()) { - auto comma_pos = filter.find_first_of(','); - const auto lparen_pos = filter.find_first_of('('); - if (lparen_pos < comma_pos) { - auto rparen_pos = filter.find_first_of(')'); - comma_pos = filter.find_first_of(',', rparen_pos); - const auto op_filter = filter.substr(0, comma_pos); - if (op_filter == op_full_name) { - return true; - } - } else { - const auto op_filter = filter.substr(0, comma_pos); - if (op_filter == op_name) { - return true; - } + if (op_names_filter == nullptr) { + return true; + } + const auto op_name = op_desc(op); + const auto op_full_name = op_name + "(" + vars() + ")"; + for (const auto & entry : op_filter_entries(op_names_filter)) { + if (entry.find_first_of('(') != std::string_view::npos) { + // a full test case string, matched exactly + if (entry == op_full_name) { + return true; } - filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : ""; + } else if (op_filter_entry_matches(entry, op_name)) { + return true; } - return false; - } else { - return true; } + return false; } test_status_t eval(ggml_backend_t backend1, @@ -1662,6 +1690,9 @@ struct test_case { total_time_us += end_time - start_time; total_mem += mem; total_runs += n_runs; + + // re-draw any data-dependent inputs (expert ids) outside the timed region + reinit_perf_iter(ctx.get()); } while (total_time_us < 1000*1000); // run for at least 1 second // Create test result @@ -2337,27 +2368,42 @@ struct test_get_rows : public test_case { const int r; // rows to get const int be1; // batch size const int be2; // batch size - const bool v; // view (non-contiguous src1) + const bool v; // view src1 + const bool vs0; // view src0 + const int offset_cols; // // column offset of the view src0 std::string vars() override { - return VARS_TO_STR7(type, n, m, r, be1, be2, v); + return VARS_TO_STR9(type, n, m, r, be1, be2, v, vs0, offset_cols); } - test_get_rows(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int be1 = 1, int be2 = 1, bool v = false) - : type(type), n(n), m(m), r(r), be1(be1), be2(be2), v(v) {} + test_get_rows(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int be1 = 1, int be2 = 1, bool v = false, bool vs0 = false, int offset_cols = 0) + : type(type), n(n), m(m), r(r), be1(be1), be2(be2), v(v), vs0(vs0), offset_cols(offset_cols) {} ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * in = ggml_new_tensor_4d(ctx, type, n, m, be1, be2); - ggml_set_name(in, "in"); + ggml_tensor * in; + if (vs0) { + const int offset_rows = 3; + const int padded_m = m + offset_rows; + const int padded_n = n + offset_cols; + ggml_tensor * in_padded = ggml_new_tensor_4d(ctx, type, padded_n, padded_m, be1, be2); + ggml_set_name(in_padded, "in_padded"); + in = ggml_view_4d(ctx, in_padded, n, m, be1, be2, + in_padded->nb[1], in_padded->nb[2], in_padded->nb[3], + offset_cols * in_padded->nb[0] + offset_rows * in_padded->nb[1]); + ggml_set_name(in, "in_view"); + } else { + in = ggml_new_tensor_4d(ctx, type, n, m, be1, be2); + ggml_set_name(in, "in"); + } - ggml_tensor * rows = ggml_new_tensor_3d(ctx, GGML_TYPE_I32, r, be1, be2); + ggml_tensor * rows = ggml_new_tensor_3d(ctx, GGML_TYPE_I32, v ? r + 1 : r, be1, be2); ggml_set_name(rows, "rows"); if (v) { - rows = ggml_view_3d(ctx, rows, r/2, be1, be2, rows->nb[1], rows->nb[2], 0); + rows = ggml_view_3d(ctx, rows, r/2, be1, be2, rows->nb[1], rows->nb[2], rows->nb[0]); ggml_set_name(rows, "view_of_rows"); } - const bool grad_supported = ggml_is_matrix(in) && ggml_is_vector(rows); + const bool grad_supported = !vs0 && ggml_is_matrix(in) && ggml_is_vector(rows); if (grad_supported) { ggml_set_param(in); // rows is a constant input -> no gradients @@ -2371,14 +2417,16 @@ struct test_get_rows : public test_case { void initialize_tensors(ggml_context * ctx) override { for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (ggml_is_view_op(t->op)) { + continue; + } if (t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { continue; } // rows - std::vector<int> data(r*be1*be2); - for (int i = 0; i < r*be1*be2; i++) { + std::vector<int> data(ggml_nelements(t)); + for (size_t i = 0; i < data.size(); i++) { data[i] = rand() % m; } - ggml_backend_tensor_set(t, data.data(), 0, r * be1 * be2 * sizeof(int)); + ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(int)); } else { init_tensor_uniform(t); } @@ -3181,13 +3229,16 @@ struct test_rope_set_rows : public test_case { } }; -// GGML_OP_RMS_NORM + GGML_OP_MUL + GGML_OP_ROPE (+ GGML_OP_VIEW + GGML_OP_SET_ROWS) +// GGML_OP_RMS_NORM with optional GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW and GGML_OP_SET_ROWS struct test_rms_norm_mul_rope : public test_case { const std::array<int64_t, 4> ne; const float eps; const bool multi_add; // test a sequence of adds feeding into rms_norm + const bool mul; + const bool rope; const bool set_rows; const bool broadcast; // multiply by a 1D [ne0] weight, as model norm weights are + const ggml_type set_rows_type; int mode; std::string op_desc(ggml_tensor * t) override { @@ -3198,63 +3249,90 @@ struct test_rms_norm_mul_rope : public test_case { bool run_whole_graph() override { return true; } std::string vars() override { - return VARS_TO_STR6(ne, eps, multi_add, set_rows, broadcast, mode); + return VARS_TO_STR9(ne, eps, multi_add, mul, rope, set_rows, broadcast, mode, set_rows_type); } test_rms_norm_mul_rope(std::array<int64_t, 4> ne, float eps = 1e-6f, bool multi_add = false, - bool set_rows = false, bool broadcast = false, int mode = GGML_ROPE_TYPE_NORMAL) - : ne(ne), eps(eps), multi_add(multi_add), set_rows(set_rows), broadcast(broadcast), mode(mode) {} + bool set_rows = false, bool broadcast = false, int mode = GGML_ROPE_TYPE_NORMAL, + bool mul = true, bool rope = true, ggml_type set_rows_type = GGML_TYPE_F16) + : ne(ne), eps(eps), multi_add(multi_add), mul(mul), rope(rope), set_rows(set_rows), broadcast(broadcast), + set_rows_type(set_rows_type), mode(mode) {} ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], 1); - ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], 1); - ggml_tensor * c = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], 1); + ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], ne[3]); + + ggml_tensor * b = nullptr; + ggml_tensor * c = nullptr; + ggml_tensor * w = nullptr; + if (multi_add || (mul && !broadcast)) { + b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], 1); + } if (multi_add) { - a = ggml_add(ctx, ggml_add(ctx, a, b), c); + c = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], 1); + } + if (mul) { + w = broadcast ? ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]) : b; } - ggml_tensor * w = broadcast ? ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]) : b; + if (multi_add) { + a = ggml_add(ctx, ggml_add(ctx, a, b), c); + } - a = ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), w); + a = ggml_rms_norm(ctx, a, eps); - ggml_tensor * pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne[2]); + if (mul) { + a = ggml_mul(ctx, a, w); + } - ggml_tensor * rope = ggml_rope(ctx, a, pos, ne[0], mode); + if (rope) { + const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; + ggml_tensor * pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne[2] * (is_mrope ? 4 : 1)); - ggml_tensor * out; + if (is_mrope) { + const int n_dims = ne[0]; + int sections[4] = { n_dims/3, n_dims/3, n_dims/3, 0 }; + a = ggml_rope_multi(ctx, a, pos, nullptr, n_dims, sections, mode, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); + } else { + a = ggml_rope(ctx, a, pos, ne[0], mode); + } + } if (set_rows) { - ggml_tensor * view = ggml_view_2d(ctx, rope, ne[0] * ne[1], ne[2], rope->nb[2], 0); + ggml_tensor * view = ggml_view_2d(ctx, a, ne[0] * ne[1], ne[2], a->nb[2], 0); - ggml_tensor * dst = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, ne[0] * ne[1], ne[2] * ne[3], 1, 1); + ggml_tensor * dst = ggml_new_tensor_2d(ctx, set_rows_type, ne[0] * ne[1], ne[2] * 2); ggml_set_name(dst, "dst"); - ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, GGML_TYPE_I64, ne[2], 1, 1); + ggml_tensor * row_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, ne[2]); ggml_set_name(row_idxs, "row_idxs"); - out = ggml_set_rows(ctx, dst, view, row_idxs); - ggml_set_name(out, "out"); - } else { - out = rope; + a = ggml_set_rows(ctx, dst, view, row_idxs); } - return out; + ggml_set_name(a, "out"); + return a; } void initialize_tensors(ggml_context * ctx) override { for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I64 || t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { - continue; + if (t->type == GGML_TYPE_I64) { + init_set_rows_row_ids(t, ne[2] * 2); + } else if (t->type == GGML_TYPE_I32) { + std::vector<int32_t> data(ggml_nelements(t)); + for (int32_t & value : data) { + value = rand() % 512; } - - init_set_rows_row_ids(t, ne[2]); + ggml_backend_tensor_set(t, data.data(), 0, ggml_nbytes(t)); } else { init_tensor_uniform(t); } } } + + double max_nmse_err() override { + return ne[0] == 8192 ? 5e-6 : test_case::max_nmse_err(); + } }; // GGML_OP_ARGMAX @@ -3834,6 +3912,14 @@ struct test_bin_bcast : public test_case { return op == ggml_div; } + double max_nmse_err() override { + if (op == ggml_add && type == GGML_TYPE_F16 && nf > 1) { + // Fused ADDs can keep FP32 intermediates while the CPU rounds each ADD to FP16. + return 1e-6; + } + return test_case::max_nmse_err(); + } + double max_maa_err() override { return op == ggml_add ? 1e-4 : 1e-3; } @@ -4089,6 +4175,44 @@ struct test_norm_mul_add : public test_case { return out; } }; +// GGML_OP_NORM/RMS_NORM + GGML_OP_SCALE +struct test_norm_scale : public test_case { + const ggml_type type; + const std::array<int64_t, 4> ne; + const float eps; + const bool rms; + const float scale; + + std::string vars() override { + return VARS_TO_STR5(type, ne, eps, rms, scale); + } + + test_norm_scale(ggml_type type = GGML_TYPE_F32, + std::array<int64_t, 4> ne = {64, 5, 4, 3}, + float eps = 1e-6f, + bool rms = false, + float scale = 1.5f) + : type(type), ne(ne), eps(eps), rms(rms), scale(scale) {} + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return rms ? "RMS_NORM_SCALE" : "NORM_SCALE"; + } + + bool run_whole_graph() override { return true; } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * n = rms ? ggml_rms_norm(ctx, a, eps) : ggml_norm(ctx, a, eps); + ggml_tensor * out = ggml_scale(ctx, n, scale); + ggml_set_name(out, "out"); + + return out; + } +}; + // GGML_OP_RMS_NORM struct test_rms_norm : public test_case { const ggml_type type; @@ -4179,13 +4303,16 @@ struct test_rms_norm_back : public test_case { } }; -// GGML_OP_RMS_NORM + GGML_OP_MUL + GGML_OP_ADD +// GGML_OP_RMS_NORM + GGML_OP_MUL + GGML_OP_ADD (+ GGML_OP_MUL) struct test_rms_norm_mul_add : public test_case { const ggml_type type; const std::array<int64_t, 4> ne; const float eps; const bool broadcast; const bool multi_add; // test a sequence of adds feeding into rms_norm + const bool post_mul; + const bool alias_rms_input; + const bool weight_broadcast; std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -4195,20 +4322,23 @@ struct test_rms_norm_mul_add : public test_case { bool run_whole_graph() override { return true; } std::string vars() override { - return VARS_TO_STR5(type, ne, eps, broadcast, multi_add); + return VARS_TO_STR8(type, ne, eps, broadcast, multi_add, post_mul, alias_rms_input, weight_broadcast); } test_rms_norm_mul_add(ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne = {64, 5, 4, 3}, - float eps = 1e-6f, bool broadcast = false, bool multi_add = false) - : type(type), ne(ne), eps(eps), broadcast(broadcast), multi_add(multi_add) {} + float eps = 1e-6f, bool broadcast = false, bool multi_add = false, bool post_mul = false, + bool alias_rms_input = false, bool weight_broadcast = false) + : type(type), ne(ne), eps(eps), broadcast(broadcast), multi_add(multi_add), post_mul(post_mul), + alias_rms_input(alias_rms_input), weight_broadcast(weight_broadcast) {} ggml_tensor * build_graph(ggml_context * ctx) override { std::array<int64_t, 4> broadcast_dims = {ne[0]*2, ne[1]*3, ne[2]*3, ne[3]*4}; ggml_tensor * a = ggml_new_tensor(ctx, type, 4, broadcast ? broadcast_dims.data() : ne.data()); - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_tensor * b = weight_broadcast ? ggml_new_tensor_1d(ctx, type, ne[0]) : ggml_new_tensor(ctx, type, 4, ne.data()); ggml_tensor * c = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_tensor * d = nullptr; ggml_set_param(a); ggml_set_name(a, "a"); @@ -4219,10 +4349,20 @@ struct test_rms_norm_mul_add : public test_case { // Use a, b and c early, so we don't end up with an OP_NONE between rms_norm and mul a = ggml_add(ctx, ggml_add(ctx, a, b), c); + if (post_mul) { + d = ggml_new_tensor_1d(ctx, type, 1); + ggml_set_param(d); + ggml_set_name(d, "d"); + a = ggml_add(ctx, a, d); + } if (multi_add) { a = ggml_add(ctx, ggml_add(ctx, a, b), c); } - ggml_tensor * out = ggml_add(ctx, ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), b), c); + ggml_tensor * mul = ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), b); + ggml_tensor * out = alias_rms_input ? ggml_add_inplace(ctx, a, mul) : ggml_add(ctx, mul, c); + if (post_mul) { + out = ggml_mul(ctx, out, d); + } ggml_set_name(out, "out"); return out; @@ -4390,8 +4530,7 @@ struct test_relu_sqr : public test_case { } }; -// GGML_OP_UNARY(SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL (fused operation). -// `layout` and `tail` are used for fallback cases where fusion must be skipped +// GGML_OP_UNARY(GELU|SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL (fused operation). struct test_unary_mul : public test_case { const ggml_unary_op op; const ggml_type type; @@ -4412,7 +4551,8 @@ struct test_unary_mul : public test_case { // performs; relax the tolerance to match that drift switch (type) { case GGML_TYPE_F16: return 5e-5; - default: return 1e-7; + // gelu shader uses exp form, CPU uses tanhf + default: return op == GGML_UNARY_OP_GELU ? 5e-7 : 1e-7; } } @@ -4471,17 +4611,45 @@ struct test_unary_mul : public test_case { } else if (layout == "bcast") { a = ggml_new_tensor(ctx, type, 4, ne.data()); b = ggml_new_tensor_4d(ctx, type, ne[0], 1, 1, 1); + } else if (layout == "rep_ne0") { + // repeat on dim 0 + a = ggml_new_tensor(ctx, type, 4, ne.data()); + std::array<int64_t, 4> ne_b = ne; + ne_b[0] /= 4; + b = ggml_new_tensor(ctx, type, 4, ne_b.data()); + } else if (layout == "view_mid") { + // VIEW between UNARY and MUL + a = ggml_new_tensor(ctx, type, 4, ne.data()); + b = nullptr; + } else if (layout == "gate") { + // small gate on src1 + const std::array<int64_t, 4> ne_gate = { 1, ne[1], ne[2], ne[3] }; + a = ggml_new_tensor(ctx, type, 4, ne_gate.data()); + b = ggml_new_tensor(ctx, type, 4, ne.data()); } else { GGML_ABORT("unknown layout %s", layout.c_str()); } - ggml_set_name(a, "a"); - ggml_set_name(b, "b"); + if (a != nullptr) { + ggml_set_name(a, "a"); + } + if (b != nullptr) { + ggml_set_name(b, "b"); + } ggml_tensor * u = ggml_unary(ctx, a, op); ggml_set_name(u, "unary"); // a broadcasting operand can only be the second one - const bool second = swap && layout != "bcast"; + const bool second = layout == "gate" || (swap && layout != "bcast" && layout != "view_mid"); + if (layout == "view_mid") { + std::array<int64_t, 4> ne_base = ne; + ne_base[0] *= 2; + ggml_tensor * base = ggml_new_tensor(ctx, type, 4, ne_base.data()); + ggml_set_name(base, "base"); + b = ggml_view_4d(ctx, base, ne[0], ne[1], ne[2], ne[3], + base->nb[1], base->nb[2], base->nb[3], 0); + ggml_set_name(b, "b"); + } ggml_tensor * out = second ? ggml_mul(ctx, b, u) : ggml_mul(ctx, u, b); if (tail == "reuse") { @@ -4603,6 +4771,9 @@ struct test_dsv4_hc : public test_case { if (name == "post") { lo = 0.0f; hi = 2.0f; return true; } + if (name == "gate") { + lo = -4.0f; hi = 4.0f; return true; + } if (name == "x" || name == "residual") { lo = -1.0f; hi = 1.0f; return true; } @@ -4666,7 +4837,9 @@ struct test_dsv4_hc_comb : public test_dsv4_hc { struct test_dsv4_hc_pre : public test_dsv4_hc { const int64_t n_embd; + const int64_t n_hc; const int64_t n_tokens; + const bool gated; std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -4674,20 +4847,27 @@ struct test_dsv4_hc_pre : public test_dsv4_hc { } std::string vars() override { - return VARS_TO_STR2(n_embd, n_tokens); + return VARS_TO_STR4(n_embd, n_hc, n_tokens, gated); } - test_dsv4_hc_pre(int64_t n_embd = 31, int64_t n_tokens = 17) - : n_embd(n_embd), n_tokens(n_tokens) {} + test_dsv4_hc_pre(int64_t n_embd = 31, int64_t n_hc = 4, int64_t n_tokens = 17, bool gated = false) + : n_embd(n_embd), n_hc(n_hc), n_tokens(n_tokens), gated(gated) {} ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd, hc, n_tokens); + ggml_tensor * x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd, n_hc, n_tokens); ggml_set_name(x, "x"); - ggml_tensor * weights = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hc, n_tokens); - ggml_set_name(weights, "weights"); + if (gated) { + ggml_tensor * gate = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd, n_hc, n_tokens); + ggml_set_name(gate, "gate"); - out = ggml_dsv4_hc_pre(ctx, x, weights); + out = ggml_dsv4_hc_pre_gated(ctx, x, gate, 1.0f/n_hc); + } else { + ggml_tensor * weights = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_hc, n_tokens); + ggml_set_name(weights, "weights"); + + out = ggml_dsv4_hc_pre(ctx, x, weights); + } ggml_set_name(out, "out"); return out; } @@ -4696,6 +4876,7 @@ struct test_dsv4_hc_pre : public test_dsv4_hc { struct test_dsv4_hc_post : public test_dsv4_hc { const int64_t n_embd; const int64_t n_tokens; + const bool identity; std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -4703,11 +4884,11 @@ struct test_dsv4_hc_post : public test_dsv4_hc { } std::string vars() override { - return VARS_TO_STR2(n_embd, n_tokens); + return VARS_TO_STR3(n_embd, n_tokens, identity); } - test_dsv4_hc_post(int64_t n_embd = 31, int64_t n_tokens = 17) - : n_embd(n_embd), n_tokens(n_tokens) {} + test_dsv4_hc_post(int64_t n_embd = 31, int64_t n_tokens = 17, bool identity = false) + : n_embd(n_embd), n_tokens(n_tokens), identity(identity) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens); @@ -4719,8 +4900,11 @@ struct test_dsv4_hc_post : public test_dsv4_hc { ggml_tensor * post = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hc, n_tokens); ggml_set_name(post, "post"); - ggml_tensor * comb = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, hc, hc, n_tokens); - ggml_set_name(comb, "comb"); + ggml_tensor * comb = nullptr; + if (!identity) { + comb = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, hc, hc, n_tokens); + ggml_set_name(comb, "comb"); + } out = ggml_dsv4_hc_post(ctx, x, residual, post, comb); ggml_set_name(out, "out"); @@ -5106,6 +5290,122 @@ struct test_gated_delta_net : public test_case { } }; +// GGML_OP_GATED_DELTA_NET + GGML_OP_CPY (recurrent cache fusion) +struct test_gated_delta_net_cache_fusion : public test_case { + const ggml_type type; + + const int64_t head_count; + const int64_t head_size; + const int64_t n_seq_tokens; + const int64_t n_seqs; + const int64_t K; // snapshot slot count (>1) + + ggml_tensor * cpy_node = nullptr; + + std::string vars() override { + return VARS_TO_STR6(type, head_count, head_size, n_seq_tokens, n_seqs, K); + } + + test_gated_delta_net_cache_fusion(ggml_type type = GGML_TYPE_F32, + int64_t head_count = 4, int64_t head_size = 32, int64_t n_seq_tokens = 2, int64_t n_seqs = 1, + int64_t K = 2) + : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), K(K) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t S_v = head_size; + const int64_t H_v = head_count; + const int64_t H_k = head_count; + const int64_t D = S_v * S_v * H_v; + const int64_t n_written = std::min<int64_t>(n_seq_tokens, K); + + ggml_tensor * q = ggml_new_tensor_4d(ctx, type, head_size, H_k, n_seq_tokens, n_seqs); + ggml_tensor * k = ggml_new_tensor_4d(ctx, type, head_size, H_k, n_seq_tokens, n_seqs); + ggml_tensor * v = ggml_new_tensor_4d(ctx, type, head_size, H_v, n_seq_tokens, n_seqs); + ggml_set_name(q, "q"); + ggml_set_name(k, "k"); + ggml_set_name(v, "v"); + ggml_tensor * g = ggml_new_tensor_4d(ctx, type, 1, H_v, n_seq_tokens, n_seqs); + ggml_tensor * beta = ggml_new_tensor_4d(ctx, type, 1, H_v, n_seq_tokens, n_seqs); + ggml_tensor * state = ggml_new_tensor_4d(ctx, type, head_size, head_size, H_v, n_seqs); + ggml_set_name(g, "g"); + ggml_set_name(beta, "beta"); + ggml_set_name(state, "state"); + + q = ggml_l2_norm(ctx, q, 1e-6f); + k = ggml_l2_norm(ctx, k, 1e-6f); + + ggml_tensor * gdn_out = ggml_gated_delta_net(ctx, q, k, v, g, beta, state, K); + ggml_set_name(gdn_out, "gdn_out"); + + // attn scores view (first part of the gdn output) + ggml_tensor * attn = ggml_view_4d(ctx, gdn_out, + S_v, H_v, n_seq_tokens, n_seqs, + ggml_row_size(gdn_out->type, S_v), + ggml_row_size(gdn_out->type, S_v * H_v), + ggml_row_size(gdn_out->type, S_v * H_v * n_seq_tokens), 0); + ggml_set_name(attn, "attn"); + + // snapshot tail view [D, n_seqs, n_written] + const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs; + ggml_tensor * src = ggml_view_3d(ctx, gdn_out, + D, n_seqs, n_written, + ggml_row_size(gdn_out->type, D), + ggml_row_size(gdn_out->type, D * n_seqs), + ggml_row_size(gdn_out->type, attn_score_elems)); + + // recurrent cache view [D, n_seqs, n_written] + ggml_tensor * cache = ggml_new_tensor_3d(ctx, type, D, n_seqs, n_written); + ggml_set_name(cache, "cache"); + ggml_tensor * dst = ggml_view_3d(ctx, cache, + D, n_seqs, n_written, + ggml_row_size(cache->type, D), + ggml_row_size(cache->type, D * n_seqs), 0); + + ggml_tensor * cpy = ggml_cpy(ctx, src, dst); + ggml_set_name(cpy, "gdn_cache_cpy"); + cpy_node = cpy; + + // read the cpy output (not the plain dst view, which would not pull the cpy into the graph) + // so that neither the gdn nor the cpy is the graph output + ggml_tensor * out = ggml_sum(ctx, cpy); + return out; + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "GATED_DELTA_NET_CACHE_FUSION"; + } + + bool run_whole_graph() override { return true; } + std::vector<ggml_tensor *> fusion_test_nodes() override { return { cpy_node }; } + + uint64_t op_flops(ggml_tensor * t) override { + GGML_UNUSED(t); + const uint64_t S_v = head_size; + const uint64_t H_v = head_count; + const uint64_t T = n_seq_tokens; + const uint64_t B = n_seqs; + return (4ull*S_v + 2ull*S_v*S_v) * H_v * T * B; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (ggml_is_view_op(t->op)) { continue; } + if (strcmp(t->name, "g") == 0) { + init_tensor_uniform(t, -20.0f, -1e-4f); + } else if (strcmp(t->name, "beta") == 0) { + init_tensor_uniform(t, 0.0f, 1.0f); + } else if (strcmp(t->name, "v") == 0) { + init_tensor_uniform(t, -0.3f, 5.0f); + } else if (strcmp(t->name, "cache") == 0) { + init_tensor_uniform(t, 0.0f, 0.0f); + } else { + init_tensor_uniform(t); + } + } + } +}; + // GGML_OP_GATED_LINEAR_ATTN struct test_gla : public test_case { const ggml_type type; @@ -5385,25 +5685,35 @@ struct test_turbo_wht : public test_case { } }; -static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats) { +static void init_mul_mat_id_ids(ggml_context * ctx, int n_mats) { std::random_device rd; std::default_random_engine rng(rd()); for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { continue; } - // ids - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector<int32_t> data(t->ne[0]); - for (int i = 0; i < t->ne[0]; i++) { - data[i] = i % n_mats; - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + if (t->type != GGML_TYPE_I32 || ggml_is_view_op(t->op)) { + continue; + } + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector<int32_t> data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = i % n_mats; } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + } + } +} + +static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats, float amax = 1.0f) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I32) { + continue; + } else if (amax != 1.0f && t->type == GGML_TYPE_F32) { + init_tensor_uniform(t, -amax, amax); } else { init_tensor_uniform(t); } } + init_mul_mat_id_ids(ctx, n_mats); } // GGML_OP_MUL_MAT_ID @@ -5416,9 +5726,10 @@ struct test_mul_mat_id : public test_case { const int64_t m; const int64_t n; const int64_t k; + const float amax; // magnitude of src1 std::string vars() override { - return VARS_TO_STR8(type_a, type_b, n_mats, n_used, b, m, n, k); + return VARS_TO_STR9(type_a, type_b, n_mats, n_used, b, m, n, k, amax); } double max_nmse_err() override { @@ -5448,9 +5759,10 @@ struct test_mul_mat_id : public test_case { test_mul_mat_id(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, int n_mats = 8, int n_used = 2, bool b = false, - int64_t m = 32, int64_t n = 32, int64_t k = 32) + int64_t m = 32, int64_t n = 32, int64_t k = 32, + float amax = 1.0f) : type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b), - m(m), n(n), k(k) { + m(m), n(n), k(k), amax(amax) { GGML_ASSERT(n_used <= n_mats); } @@ -5472,11 +5784,20 @@ struct test_mul_mat_id : public test_case { ggml_tensor * out = ggml_mul_mat_id(ctx, as, b, ids); ggml_set_name(out, "out"); + if (amax > 65504.0f) { + // src1 exceeds F16 range + ggml_prec_set_src(out, GGML_PREC_F32, 1); + } + return out; } void initialize_tensors(ggml_context * ctx) override { - init_mul_mat_id_tensors(ctx, n_mats); + init_mul_mat_id_tensors(ctx, n_mats, amax); + } + + void reinit_perf_iter(ggml_context * ctx) override { + init_mul_mat_id_ids(ctx, n_mats); } }; @@ -6532,13 +6853,6 @@ struct test_conv_2d : public test_case { // Whether the inputs are contiguous in the channel dim or the width dim const bool cwhn; - // If true, the direct CONV_2D will be used in the graph, otherwise it - // uses ggml_conv_2d: - // * if the program is called with -o CONV_2D_DIRECT_IMPL, the - // CONV_2D graph will be built, while - // * if the program is called with -o CONV_2D_INDIRECT_IMPL, the - // IM2COL -> MUL_MM graph will be built. - std::string vars() override { return VARS_TO_STR10(ne_input, ne_kernel, type_kernel, stride0, stride1, padding0, padding1, dilation0, dilation1, cwhn); } @@ -7154,7 +7468,7 @@ struct test_topk_moe : public test_case { } }; -struct test_moe_weighted_reduction : public test_case { +struct test_moe_reduce : public test_case { const int64_t n_embd; const int64_t n_expert_used; const int64_t n_tokens; @@ -7162,7 +7476,7 @@ struct test_moe_weighted_reduction : public test_case { const bool with_expert_scale; const bool interleaved_views_adds; - test_moe_weighted_reduction( + test_moe_reduce( int64_t n_embd, int64_t n_expert_used, int64_t n_tokens, bool unaligned_experts = false, bool with_expert_scale = false, bool interleaved_views_adds = false) : n_embd(n_embd), n_expert_used(n_expert_used), n_tokens(n_tokens), @@ -7175,7 +7489,7 @@ struct test_moe_weighted_reduction : public test_case { std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); - return "MOE_WEIGHTED_REDUCTION"; + return "MOE_REDUCE"; } bool run_whole_graph() override { return true; } @@ -7222,7 +7536,7 @@ struct test_moe_weighted_reduction : public test_case { ggml_build_forward_expand(gf, out); } } - ggml_set_name(out, "moe_weighted_reduction"); + ggml_set_name(out, "moe_reduce"); return out; } }; @@ -7501,20 +7815,32 @@ struct test_sum_rows : public test_case { struct test_mean : public test_case { const ggml_type type; const std::array<int64_t, 4> ne; + const bool permute; + const bool slice; std::string vars() override { - return VARS_TO_STR2(type, ne); + return VARS_TO_STR4(type, ne, permute, slice); } test_mean(ggml_type type = GGML_TYPE_F32, - std::array<int64_t, 4> ne = {10, 5, 4, 3}) - : type(type), ne(ne) {} + std::array<int64_t, 4> ne = {10, 5, 4, 3}, + bool permute = false, bool slice = false) + : type(type), ne(ne), permute(permute), slice(slice) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); ggml_set_param(a); ggml_set_name(a, "a"); + if (slice) { + a = ggml_view_4d(ctx, a, + ne[0], ne[1], ne[2] / 2, ne[3] - 1, + a->nb[1], a->nb[2] * 2, a->nb[3], /*offset=*/a->nb[3]); + } + if (permute) { + a = ggml_permute(ctx, a, 0, 2, 3, 1); + } + ggml_tensor * out = ggml_mean(ctx, a); ggml_set_name(out, "out"); @@ -7688,6 +8014,49 @@ struct test_sinkhorn_norm : public test_case { } }; +// GGML_OP_L2_NORM x N: independent same-shape norms in one graph (strided qkv views or +// contiguous), consuming adds nested so the norms stay adjacent in the graph. +struct test_l2_norm_batch : public test_case { + const ggml_type type; + const std::array<int64_t, 4> ne; + const int n_norms; + const float eps; + const bool strided; + + std::string vars() override { return VARS_TO_STR5(type, ne, n_norms, eps, strided); } + std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); return "L2_NORM_BATCH"; } + bool run_whole_graph() override { return true; } + + test_l2_norm_batch(ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne = { 128, 16, 16, 1 }, + int n_norms = 4, float eps = 1e-12f, bool strided = true) + : type(type), ne(ne), n_norms(n_norms), eps(eps), strided(strided) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + GGML_ASSERT(n_norms >= 2 && n_norms <= 8); + ggml_tensor * parent = nullptr; + if (strided) { + parent = ggml_new_tensor_4d(ctx, type, ne[0], ne[1] * n_norms, ne[2], ne[3]); // qkv buffer + } + ggml_tensor * norms[8] = {}; + for (int t = 0; t < n_norms; ++t) { + ggml_tensor * src; + if (strided) { + src = ggml_view_4d(ctx, parent, ne[0], ne[1], ne[2], ne[3], parent->nb[1], parent->nb[2], + parent->nb[3], t * ne[1] * parent->nb[1]); + } else { + src = ggml_new_tensor(ctx, type, 4, ne.data()); + } + norms[t] = ggml_l2_norm(ctx, src, eps); + } + ggml_tensor * out = norms[n_norms - 1]; + for (int t = n_norms - 2; t >= 0; --t) { + out = ggml_add(ctx, norms[t], out); + } + ggml_set_name(out, "out"); + return out; + } +}; + // GGML_OP_L2_NORM struct test_l2_norm : public test_case { const ggml_type type; @@ -8098,7 +8467,7 @@ struct test_flash_attn_ext : public test_case { ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf(hsk), max_bias, logit_softcap); ggml_flash_attn_ext_add_sinks(out, s); ggml_flash_attn_ext_set_n_kv_max(out, n_kv_max); - ggml_flash_attn_ext_set_prec (out, prec); + ggml_prec_set_acc(out, prec); ggml_set_name(out, "out"); return out; @@ -9191,7 +9560,7 @@ static const ggml_type all_types[] = { GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_TQ2_0, - // GGML_TYPE_TQ1_0, // TODO: implement for all backends + GGML_TYPE_TQ1_0, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, @@ -9219,7 +9588,7 @@ static const ggml_type other_types[] = { GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_TQ2_0, - // GGML_TYPE_TQ1_0, // TODO: implement for all backends + GGML_TYPE_TQ1_0, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, @@ -9256,7 +9625,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } // fused unary + mul (gated activations that are not expressed as GGML_OP_GLU) - for (ggml_unary_op op : { GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) { + for (ggml_unary_op op : { GGML_UNARY_OP_GELU, GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) { for (ggml_type type : { GGML_TYPE_F16, GGML_TYPE_F32 }) { for (bool swap : { false, true }) { test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, swap)); @@ -9267,9 +9636,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, true, "pad_other")); test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, true, "halves")); test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "packed", "consumer")); + test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "bcast")); + test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "rep_ne0")); + test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "view_mid")); + test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "gate")); // must not fuse test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "strided_dim1")); - test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "bcast")); test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "packed", "reuse")); } } @@ -9291,16 +9663,29 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_dsv4_hc_comb(17, 4)); test_cases.emplace_back(new test_dsv4_hc_comb(257, 8)); test_cases.emplace_back(new test_dsv4_hc_comb(17, 20)); + // production n_iter (DeepSeek-V4 uses 20) across batch sizes that cross + // subgroup and workgroup boundaries; 1 = single-token decode + for (int64_t n_tokens : {1, 256, 336, 512, 513, 1024, 2048}) { + test_cases.emplace_back(new test_dsv4_hc_comb(n_tokens, 20)); + } - test_cases.emplace_back(new test_dsv4_hc_pre(1, 1)); - test_cases.emplace_back(new test_dsv4_hc_pre(31, 17)); - test_cases.emplace_back(new test_dsv4_hc_pre(128, 257)); - test_cases.emplace_back(new test_dsv4_hc_pre(4096, 21)); + test_cases.emplace_back(new test_dsv4_hc_pre(1, 4, 1)); + test_cases.emplace_back(new test_dsv4_hc_pre(31, 4, 17)); + test_cases.emplace_back(new test_dsv4_hc_pre(128, 4, 257)); + test_cases.emplace_back(new test_dsv4_hc_pre(4096, 4, 21)); + test_cases.emplace_back(new test_dsv4_hc_pre(31, 4, 17, true)); + test_cases.emplace_back(new test_dsv4_hc_pre(4096, 4, 21, true)); + for (int64_t n_hc : {1, 2, 3, 5, 8, 65}) { + test_cases.emplace_back(new test_dsv4_hc_pre(128, n_hc, 17)); + test_cases.emplace_back(new test_dsv4_hc_pre(128, n_hc, 17, true)); + } test_cases.emplace_back(new test_dsv4_hc_post(1, 1)); test_cases.emplace_back(new test_dsv4_hc_post(31, 17)); test_cases.emplace_back(new test_dsv4_hc_post(128, 257)); test_cases.emplace_back(new test_dsv4_hc_post(4096, 21)); + test_cases.emplace_back(new test_dsv4_hc_post(31, 17, true)); + test_cases.emplace_back(new test_dsv4_hc_post(4096, 21, true)); // glu ops for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { @@ -9347,15 +9732,20 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { for (ggml_type type : all_types) { for (int b : {1, 7}) { for (bool v : {false, true}) { - test_cases.emplace_back(new test_get_rows(type, 256, 5, 4, b, 1, v)); + for (bool vs0 : {false, true}) { + test_cases.emplace_back(new test_get_rows(type, 256, 5, 4, b, 1, v, vs0)); + } } } } for (int b : {1, 7}) { for (bool v : {false, true}) { - test_cases.emplace_back(new test_get_rows(GGML_TYPE_I32, 256, 5, 4, b, 1, v)); + for (bool vs0 : {false, true}) { + test_cases.emplace_back(new test_get_rows(GGML_TYPE_I32, 256, 5, 4, b, 1, v, vs0)); + } } } + test_cases.emplace_back(new test_get_rows(GGML_TYPE_F32, 256, 8, 2, 1, 1, false, true, 3)); test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 8, 2, 1, false)); test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 70000, 4, 1, false)); // row count > CUDA grid-y limit (65535) @@ -9398,7 +9788,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, true)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, true)); - for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION }) { + for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION, GGML_ROPE_TYPE_IMROPE }) { for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { for (int ne2 : {1, 8, 512}) { test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 128, 32, ne2, 1 }, mode)); @@ -9406,6 +9796,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } } + test_cases.emplace_back(new test_rope_set_rows(GGML_TYPE_F32, GGML_TYPE_I32, { 128, 32, 8, 1 }, GGML_ROPE_TYPE_IMROPE)); for (ggml_type type_input : {GGML_TYPE_F32}) { for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) { @@ -9614,6 +10005,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_conv_2d({ 256, 256, 192, 1 }, { 3, 3, 192, 96 }, kernel_type, 1, 1, 1, 1, 1, 1, false)); // bool cwhn = false test_cases.emplace_back(new test_conv_2d({ 256, 256, 192, 1 }, { 3, 3, 192, 96 }, kernel_type, 1, 1, 1, 1, 1, 1, true)); // bool cwhn = true } + test_cases.emplace_back(new test_conv_2d({ 19, 17, 8, 2 }, { 3, 3, 8, 65 }, GGML_TYPE_F16, 1, 1, 1, 1, 1, 1)); + test_cases.emplace_back(new test_conv_2d({ 19, 17, 16, 3 }, { 3, 3, 16, 33 }, GGML_TYPE_F16, 2, 3, 4, 2, 2, 1)); + test_cases.emplace_back(new test_conv_2d({ 13, 11, 16, 3 }, { 1, 1, 16, 33 }, GGML_TYPE_F16, 1, 1, 0, 0, 1, 1)); // sycl backend will limit task global_range < MAX_INT // test cases for 2D im2col with large input W and H (occurs in stable-diffusion) @@ -9986,6 +10380,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, v, eps)); } test_cases.emplace_back(new test_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, false, eps, true)); + test_cases.emplace_back(new test_norm_scale(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false, 1.5f)); + test_cases.emplace_back(new test_norm_scale(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true, 1.5f)); test_cases.emplace_back(new test_rms_norm_back(GGML_TYPE_F32, { n, 5, 4, 3 }, eps)); test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false)); test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true)); @@ -10002,6 +10398,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { // dispatch across more than one grid dimension. Every case above is // small enough to hide an index reconstructed from only one of them. test_cases.emplace_back(new test_sinkhorn_norm(GGML_TYPE_F32, { 4, 4, 1024, 1 }, 1e-6f, 20)); + // sibling batching: strided (production shape) and contiguous, 2 and 4 wide + test_cases.emplace_back(new test_l2_norm_batch(GGML_TYPE_F32, { n, 5, 4, 3 }, 2, eps, true)); + test_cases.emplace_back(new test_l2_norm_batch(GGML_TYPE_F32, { n, 5, 4, 3 }, 4, eps, true)); + test_cases.emplace_back(new test_l2_norm_batch(GGML_TYPE_F32, { n, 5, 4, 3 }, 4, eps, false)); } // row lengths that are not a multiple of 32, for the scalar (33) and float4 (132, 260) paths for (uint32_t n : { 33, 132, 260 }) { @@ -10015,6 +10415,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { // in-place tests test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {64, 5, 4, 3}, false, 1e-6f, true)); + for (ggml_type set_rows_type : { GGML_TYPE_F32, GGML_TYPE_F16 }) { + test_cases.emplace_back(new test_rms_norm_mul_rope({ 256, 1, 1, 1 }, 1e-6f, false, true, false, GGML_ROPE_TYPE_NORMAL, false, false, set_rows_type)); + test_cases.emplace_back(new test_rms_norm_mul_rope({ 128, 4, 3, 1 }, 1e-6f, false, true, false, GGML_ROPE_TYPE_NORMAL, false, false, set_rows_type)); + } + for (float eps : { 0.0f, 1e-6f, 1e-4f, 1e-1f, 1.0f }) { for (uint32_t n : { 64, 1025 }) { test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false)); @@ -10040,10 +10445,20 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_add_add(GGML_TYPE_F16, GGML_TYPE_F32, { n, 5, 4, 3 }, true, false)); } + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 1536, 1, 1, 1 }, 1e-6f, false, false, true)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 256, 4, 1, 1 }, 1e-6f, false, false, true)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 256, 4, 3, 2 }, 1e-6f, false, false, true)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 256, 4, 3, 2 }, 1e-6f, false, false, true, false, true)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 1536, 1, 1, 1 }, 1e-6f, false, false, false, true)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 256, 4, 1, 1 }, 1e-6f, false, false, false, true)); + + test_cases.emplace_back(new test_rms_norm_mul_rope({128, 4, 7, 2})); + test_cases.emplace_back(new test_rms_norm_mul_rope({128, 4, 7, 2}, 1e-6f, false, true)); + for (auto multi_add : {false, true}) { for (auto set_rows : {false, true}) { for (auto broadcast : {false, true}) { - for (auto rope : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { + for (auto rope : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_IMROPE}) { test_cases.emplace_back(new test_rms_norm_mul_rope({768, 1, 1, 1}, 1e-6f, multi_add, set_rows, broadcast, rope)); test_cases.emplace_back(new test_rms_norm_mul_rope({768, 3, 1, 1}, 1e-6f, multi_add, set_rows, broadcast, rope)); test_cases.emplace_back(new test_rms_norm_mul_rope({768, 3, 5, 1}, 1e-6f, multi_add, set_rows, broadcast, rope)); @@ -10130,6 +10545,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 512, 256)); // many rows test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 32, 1, 32)); // too small (N<64) test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 1024, 1, 1024)); // too big (N>512) + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 64, 1, 64)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 128, 1, 128)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 256, 1, 256)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 512, 1, 512)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 128, 32, 128)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 128, 4, 128, {2, 3})); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 256, 512, 256)); // many rows // TURBO_WHT tests (GGML_OP_TURBO_WHT forward + inverse, d=128, rows 1 and 32) test_cases.emplace_back(new test_turbo_wht(0, 128, 1, 128)); // forward, 1 row @@ -10509,9 +10931,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } // m == 1, with n on both sides of MMVF_MAX_BATCH_SIZE (8): mmvf below, operand swap above - for (int64_t n : {1, 7, 8, 9, 16, 128, 512}) { + for (int64_t n : {1, 7, 8, 9, 16, 127, 128, 511, 512}) { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1, n, 2048, {1, 1}, {1, 1})); } + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1, 512, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 512, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1, 509, 2051, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 509, 2051, {1, 1}, {1, 1})); + + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 31, 509, 2051, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 32, 509, 2112, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 32, 509, 2112, {1, 1}, {1, 1})); #if 0 { @@ -10592,6 +11022,19 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 3, 2, 1})); + // token-tile boundary coverage. With n_used == n_mats every token routes to every expert, so + // each expert receives exactly n rows, with no dependence on the random draw. mul_mm_id is used + // from 32 tokens up: n = 32, 33, 47, 48, 49 reach it, leaving a last tile of 32, 1, 15, 16 and + // 17 rows - 16 and 17 straddle the point where the upper half stops being skipped. The smaller + // n cover the same row counts on the mat-vec path. + for (ggml_type type_a : {GGML_TYPE_Q4_K, GGML_TYPE_IQ2_XS, GGML_TYPE_F16}) { + for (int n : {1, 15, 16, 17, 31, 32, 33, 47, 48, 49}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 4, false, 512, n, 256)); + } + // experts that receive no rows at all + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 8, 1, false, 512, 1, 256)); + } + for (ggml_type type_a : other_types) { for (ggml_type type_b : {GGML_TYPE_F32}) { if (ggml_blck_size(type_a) != 256) { @@ -10711,8 +11154,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { // gpt-oss issue with Vulkan mmq_id test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_MXFP4, GGML_TYPE_F32, 32, 2, false, 2880, 32, 2880)); + // more than 256 experts (hoisted row-id path): 512 as in Qwen3.8-Flash-Next, + // and 1024 at the LLAMA_MAX_EXPERTS limit + for (int n : {1, 5, 64, 300}) { + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_IQ3_S, GGML_TYPE_F32, 512, 10, false, 128, n, 512)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_0, GGML_TYPE_F32, 512, 10, false, 256, n, 128)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_IQ3_S, GGML_TYPE_F32, 1024, 10, false, 128, n, 512)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_0, GGML_TYPE_F32, 1024, 10, false, 256, n, 128)); + } test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_0, GGML_TYPE_F32, 32, 2, false, 2880, 32, 2880)); + // multiple blocks per row: exercises the block-stride loop and the + // per-expert base offset, which k == 256 alone leaves untested + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 28, 10, false, 1024, 1, 4096)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 128, 8, false, 1024, 1, 2048)); + for (ggml_type type_a : all_types) { test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 2, false, 64, 16, 3*ggml_blck_size(type_a))); } @@ -10723,6 +11179,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 4, false, 16, 10, 256)); } + // test src1 f16 overflow + for (int n : {16, 32, 64}) { + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 4, false, 4096, n, 2048, 1e5f)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8, 2, false, 512, n, 256, 1e5f)); + } + for (ggml_type type_a : base_types) { for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) { for (int n_mats : {4, 8}) { @@ -10907,6 +11369,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {200001, 2, 3, 1}, true, true, GGML_TYPE_F16, {1, 1}, 0.1f, 8.0f)); test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {200000, 1, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {200000, 4, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {4, 1, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {4, 1023, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {643251, 3, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); for (float max_bias : {0.0f, 8.0f}) { @@ -11011,6 +11475,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, 1.4245f, 0.7465f, 1.4245f, false, 0, true, true, 32)); } + // Real-model RoPE: F32 forward, packed Q, 512-token prefill. + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global + for (int v : { 0, 1, 2, 3 }) { for (int dim : { 0, 1, 2, 3, }) { test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v)); @@ -11130,6 +11600,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 32, 1, 1, 1 })); test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 32, 256, 1, 1 })); test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 32768, 1, 1, 1 })); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, false)); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, false, true)); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, true)); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1024, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 })); @@ -11262,7 +11735,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { for (int hsk : { 40, 64, 72, 80, 96, 128, 192, 256, 320, 512, 576 }) { for (int hsv : { 40, 64, 72, 80, 96, 128, 192, 256, 512 }) { - if (hsk != 192 && hsk != 320 && hsk != 576 && hsk != hsv) continue; + if (hsk != 96 && hsk != 192 && hsk != 320 && hsk != 576 && hsk != hsv) continue; + if (hsk == 96 && (hsv != 64 && hsv != 96)) continue; // MiniCPM3 if (hsk == 192 && (hsv != 128 && hsv != 192)) continue; if (hsk == 576 && hsv != 512) continue; // DeepSeek MLA if (hsk == 320 && hsv != 256) continue; // Mistral4 MLA @@ -11314,6 +11788,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } } + // asymmetric head_dim (hsk != hsv) with one or both sides not 64-aligned + test_cases.emplace_back(new test_flash_attn_ext(72, 64, 4, {1, 1}, 256, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(64, 72, 4, {1, 1}, 256, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // mixed quant and Q1_0 test cases test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_F16)); @@ -11340,9 +11818,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 16384, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); // MLA shape: the V cache is a sub-view of the K cache, with quantized KV - test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); - test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); - test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {8, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {8, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {8, 1}, 1024, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); // Sparse mask hint: supported decode/prefill layouts and dense fallbacks. test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512)); @@ -11364,10 +11842,18 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512)); test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048)); + // sparse attn (qwen4 shape - gqa 12) + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, 8192, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 1, {12, 2}, 8192, 67, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512)); + // sparse mask + quantized cache test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false, 512)); test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false, 512)); + // Qwen QSA: 256/256, gqa 12, budget 2048. + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, 8192, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048)); + // more V-is-sub-view-of-K cases: other head shapes, and full views with equal head sizes test_cases.emplace_back(new test_flash_attn_ext(320, 256, 1, {32, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); test_cases.emplace_back(new test_flash_attn_ext(192, 128, 4, {8, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); @@ -11383,6 +11869,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); } + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 512, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 16, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {2, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {4, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // dense-allocated (non-view) quant K/V at batch >= 64, in cache and native layouts test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); @@ -11492,12 +11985,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } // Cover the supported boundaries, common k = 8 shapes, interleaved views and adds, and k = 16 fallback. - test_cases.emplace_back(new test_moe_weighted_reduction(63, 2, 17)); - test_cases.emplace_back(new test_moe_weighted_reduction(2048, 8, 128)); - test_cases.emplace_back(new test_moe_weighted_reduction(2048, 8, 128, false, true)); - test_cases.emplace_back(new test_moe_weighted_reduction(63, 12, 33, true, true, true)); - test_cases.emplace_back(new test_moe_weighted_reduction(2048, 15, 40, false, true)); - test_cases.emplace_back(new test_moe_weighted_reduction(2048, 16, 32, false, true)); + test_cases.emplace_back(new test_moe_reduce(63, 2, 17)); + test_cases.emplace_back(new test_moe_reduce(2048, 8, 128)); + test_cases.emplace_back(new test_moe_reduce(2048, 8, 128, false, true)); + test_cases.emplace_back(new test_moe_reduce(63, 12, 33, true, true, true)); + test_cases.emplace_back(new test_moe_reduce(2048, 15, 40, false, true)); + test_cases.emplace_back(new test_moe_reduce(2048, 16, 32, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1)); @@ -11567,6 +12060,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 32, 8, 1, 1, false, false, /*K=*/3)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 16, 2, 1, false, false, /*K=*/4)); + // gdn + cache cpy fusion (K > 1) + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 2, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 64, 4, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 4, 1, 4)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 8, 32, 4, 2, 4)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 8, 1, 4)); + #if 0 // these tests are disabled to save execution time, sbut they can be handy for debugging test_cases.emplace_back(new test_llama(2, true)); @@ -11787,6 +12287,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { GGML_TYPE_F32, {n_kv, 512, 64, 1}, false, {2, 1, 0, 3})); } + // LEAKY_RELU at FFN activation width, for direct comparison with RELU + for (int64_t n_tokens : {512, 2048}) { + test_cases.emplace_back(new test_leaky_relu(GGML_TYPE_F32, { 17408, n_tokens, 1, 1 }, 0.1f)); + } + // Conv2d: K=CRS=NPQ=4096 matmul performance uint32_t iwh_idx = 0; uint32_t kwh_idx = 1; @@ -11961,7 +12466,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { // qwen3-30b-a3b for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { - for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) { + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_XS}) { for (ggml_type type_b : {GGML_TYPE_F32}) { test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, 128, 8, false, 768, bs, 2048)); test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 128, 8, false, 768, bs, 2048, 1)); @@ -11970,7 +12475,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { } for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { - for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) { + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_XS}) { for (ggml_type type_b : {GGML_TYPE_F32}) { test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, 32, 4, false, 1792, bs, 2048)); test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 32, 4, false, 1792, bs, 2048, 1)); @@ -12004,6 +12509,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { // Qwen3-VL-8B https://github.com/ggml-org/llama.cpp/issues/17012 test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // Sparse flash attention (n_kv_max hint) decode across KV depths. + // Shapes: 576/512 DeepSeek MLA, 512/512 DeepSeek-V4/GLM-5.2, 256/256 gqa12 Qwen QSA. + for (int64_t kv : {4096, 16384, 32768}) { + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 512)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048)); + } + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); @@ -12018,6 +12531,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { // sparse decode at long context test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 0)); test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048)); + // gemma-4-26b-a4b global-attn layers: head_count_kv=2, 16 query heads (gqa_ratio=8) + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 2, { 8, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 0)); + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 2, { 8, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048)); test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 0)); test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 2048)); @@ -12036,6 +12552,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 16384, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 16384, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 65536, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 65536, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 131072, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 131072, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + for (int kv : { 4096, 8192, 16384,32768, 65536, }) { for (int hs : { 64, 128, 256, 576, }) { const int hsv = hs == 576 ? 512 : hs; @@ -12096,6 +12621,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { } } + // Real-model RoPE: F32 forward, packed Q, 512-token prefill. + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global + std::vector<std::array<int64_t, 4>> reduce_rows_cases = { { 8192, 1, 1, 1 }, { 8192, 8192, 1, 1 }, @@ -12127,6 +12658,30 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { } } + // qwen4exp sparse-attention indexer: nrows = n_tokens/n_stream, so tg gives nrows==1. + // Sweep nrows to expose how much of the device a single row leaves idle. + for (auto cols : {8192, 32768, 131072}) { + for (auto nrows : {1, 2, 4, 8, 16, 32}) { + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {cols, nrows, 1, 1}, 2048)); + } + } + // backend sampler: one row of the vocab (llama-sampler.cpp top_k) + for (auto k : {20, 40}) { + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {151936, 1, 1, 1}, k)); + } + + // short rows, many of them: MoE routing and group selection. The opposite corner from + // the indexer, and the one where a work-group per row is the wasteful choice. + for (auto cols : {2, 16, 128, 1024}) { + for (auto nrows : {1024, 8192}) { + for (auto k : {1, 2, 8, 16, 32}) { + if (k <= cols) { + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {cols, nrows, 1, 1}, k)); + } + } + } + } + for (auto nrows : {1, 4, 8, 16}) { for (auto cols : {128, 1024, 4096, 8192, 16384, 32768, 65536, 131072, 200000, 2000000}) { test_cases.emplace_back(new test_cumsum(GGML_TYPE_F32, {cols, nrows, 1, 1})); @@ -12197,6 +12752,16 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() { } } + // launch-overhead isolation: single L2_NORM launch vs batched siblings at the GDN + // production shape (strided qkv views) -- perf-mode only, the eval list has its own + // 2/4-wide coverage + for (int n : { 128, 256 }) { + test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 16, 16, 1 }, 1e-12f, false, false)); + test_cases.emplace_back(new test_l2_norm_batch(GGML_TYPE_F32, { n, 16, 16, 1 }, 2, 1e-12f, true)); + test_cases.emplace_back(new test_l2_norm_batch(GGML_TYPE_F32, { n, 16, 16, 1 }, 4, 1e-12f, true)); + } + + return test_cases; } @@ -12281,25 +12846,16 @@ static std::vector<int> fa_vec_legal_ne(int dk, int dv) { } static bool op_names_filter_selects(const char * op_names_filter, const char * op_name) { - if (!op_names_filter) { + if (op_names_filter == nullptr) { return true; } - std::string_view filter(op_names_filter); - while (!filter.empty()) { - auto comma_pos = filter.find_first_of(','); - const auto lparen_pos = filter.find_first_of('('); - std::string_view entry; - if (lparen_pos < comma_pos) { - const auto rparen_pos = filter.find_first_of(')'); - comma_pos = filter.find_first_of(',', rparen_pos); - entry = filter.substr(0, lparen_pos); - } else { - entry = filter.substr(0, comma_pos); - } - if (entry == op_name) { + for (const auto & entry : op_filter_entries(op_names_filter)) { + // a full test case string is matched by its op name prefix + const auto lparen_pos = entry.find_first_of('('); + const auto op_entry = lparen_pos != std::string_view::npos ? entry.substr(0, lparen_pos) : entry; + if (op_filter_entry_matches(op_entry, op_name)) { return true; } - filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : ""; } return false; } @@ -12307,6 +12863,11 @@ static bool op_names_filter_selects(const char * op_names_filter, const char * o // Covers padded rows, sinks, kvpad, multi-SIMDgroup reduction, quantized K/V, and MLA views. // The override is backend-global, so this runs after all parallel workers have joined. static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu, const char * op_names_filter) { + const char * LLAMA_TEST_FA_VEC_DISABLE = getenv("LLAMA_TEST_FA_VEC_DISABLE"); + if (LLAMA_TEST_FA_VEC_DISABLE) { + return true; + } + if (!op_names_filter_selects(op_names_filter, "FLASH_ATTN_EXT")) { return true; } @@ -12319,13 +12880,16 @@ static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu, return true; // not the Metal backend: nothing to force } + printf("Running FA vec slice tests (env LLAMA_TEST_FA_VEC_DISABLE=1 to skip)\n"); + struct shape_t { int dk, dv; }; const shape_t shapes[] = { { 128, 128 }, { 576, 512 } }; // mainstream head size + MLA shared K/V view const int ne01_pts[] = { 1, 3 }; // decode, and padded rows for Q=2 and Q=4 const int ne11_pts[] = { 512, 4097 }; // nsg=1, and nsg>=2 together with kvpad const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0 }; - int n_run = 0, n_fail = 0; + int n_run = 0; + int n_fail = 0; for (auto s : shapes) { for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { for (int Q : { 1, 2, 4 }) { @@ -12645,20 +13209,29 @@ static void show_test_coverage() { } static void usage(char ** argv) { - printf("Usage: %s [mode] [-o <op,..>] [-b <backend>] [-p <params regex>] [--output <console|sql|csv>] [--list-ops]", argv[0]); - printf(" [--show-coverage] [--test-file <path>] [-j <n>]\n"); - printf(" valid modes:\n"); - printf(" - test (default, compare with CPU backend for correctness)\n"); - printf(" - grad (compare gradients from backpropagation with method of finite differences)\n"); - printf(" - perf (performance evaluation)\n"); - printf(" - support (probe backend operation support)\n"); - printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n"); - printf(" optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n"); - printf(" --output specifies output format (default: console, options: console, sql, csv)\n"); - printf(" --list-ops lists all available GGML operations\n"); - printf(" --show-coverage shows test coverage\n"); - printf(" --test-file reads test operators from a test file generated by test-export-graph-ops\n"); - printf(" -j <n> runs tests using <n> parallel worker threads (default: 1, test mode only)\n"); + printf("Usage: %s [mode] [options]\n\n", argv[0]); + printf("Valid modes:\n"); + printf(" test (default) compare with CPU backend for correctness\n"); + printf(" grad compare gradients from backpropagation with method of finite differences\n"); + printf(" perf performance evaluation\n"); + printf(" support probe backend operation support\n\n"); + printf("Options:\n"); + printf(" -o <op|regex,..> comma separated list of exact op names (as given by ggml_op_desc()),\n"); + printf(" full test case strings, and/or regexes matched against the op name\n"); + printf(" -b <backend> run tests on the given backend (e.g. CPU, MTL0, CUDA0)\n"); + printf(" -p <params regex> filter test cases by a regex matched against their params\n"); + printf(" --output <console|sql|csv> output format (default: console)\n"); + printf(" --list-ops list all available GGML operations\n"); + printf(" --show-coverage show test coverage\n"); + printf(" --test-file <path> read test operators from a test file generated by test-export-graph-ops\n"); + printf(" -j <n> run tests using <n> parallel worker threads (default: 1, test mode only)\n\n"); + printf("Examples:\n"); + printf(" %s -j 8\n", argv[0]); + printf(" %s -o ADD,MUL_MAT\n", argv[0]); + printf(" %s -o ADD -p 'type=f16.*perm1=0'\n", argv[0]); + printf(" %s -b MTL0 -o 'DSV4.*'\n", argv[0]); + printf(" %s -b CUDA0 -o 'ADD(type=f16,ne=[1,1,1,1],nr=[32,1,1,1],nf=1,perm1=0,src_overlap=0)'\n", argv[0]); + printf(" %s perf -o 'MUL_MAT.*'\n", argv[0]); } int main(int argc, char ** argv) { @@ -12776,8 +13349,7 @@ int main(int argc, char ** argv) { ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); if (ggml_backend_set_n_threads_fn) { - // TODO: better value for n_threads - ggml_backend_set_n_threads_fn(backend.get(), N_THREADS); + ggml_backend_set_n_threads_fn(backend.get(), std::max<int>(1, N_THREADS/2)); } size_t free, total; // NOLINT diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 793891394ce6..36e11a30bbdb 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -23,6 +23,7 @@ static void test_command7_parser_compare(testing & t); static void test_prefix_tool_names(testing & t); static void test_tagged_peg_parser(testing & t); static void test_permute(testing & t); +static void test_invalid_utf8(testing & t); int main(int argc, char * argv[]) { testing t(std::cout); @@ -42,6 +43,7 @@ int main(int argc, char * argv[]) { t.test("prefix tool names", test_prefix_tool_names); t.test("tagged peg parser", test_tagged_peg_parser); t.test("permute", test_permute); + t.test("invalid utf8", test_invalid_utf8); return t.summary(); } @@ -358,11 +360,6 @@ static void test_example_native(testing & t) { auto parser = build_parser(tc); auto lazy = !tc.tools.empty() && tc.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; auto grammar = build_grammar([&](const common_grammar_builder & builder) { - for (const auto & def : tc.tools) { - auto function = def.at("function"); - auto parameters = function.at("parameters"); - builder.resolve_refs(parameters); - }; parser.build_grammar(builder, lazy); }); @@ -440,11 +437,6 @@ static void test_example_qwen3_coder(testing & t) { }); auto grammar = build_grammar([&](const common_grammar_builder & builder) { - for (const auto & def : tools) { - auto function = def.at("function"); - auto parameters = function.at("parameters"); - builder.resolve_refs(parameters); - }; parser.build_grammar(builder); }); @@ -513,11 +505,6 @@ static void test_example_qwen3_non_coder(testing & t) { }); auto grammar = build_grammar([&](const common_grammar_builder & builder) { - for (const auto & def : tools) { - auto function = def.at("function"); - auto parameters = function.at("parameters"); - builder.resolve_refs(parameters); - }; parser.build_grammar(builder); }); @@ -1084,3 +1071,36 @@ static void test_permute(testing & t) { )""", gbnf_of(parser)); }); } + +static void test_invalid_utf8(testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return "<think>" + p.reasoning(p.until("</think>")) + "</think>" + p.content(p.rest()) + p.end(); + }); + + t.test("replaced in reasoning and content", [&](testing & t) { + std::string input("<think>plan\xFF\xFE</think>caf\xC3\xA9 \x80 done"); + common_peg_parse_context ctx(input); + auto result = parser.parse(ctx); + t.assert_true("success", result.success()); + + common_chat_msg msg; + auto mapper = common_chat_peg_mapper(msg); + mapper.from_ast(ctx.ast, result); + + t.assert_equal("reasoning", "plan\xEF\xBF\xBD\xEF\xBF\xBD", msg.reasoning_content); + t.assert_equal("content", "caf\xC3\xA9 \xEF\xBF\xBD done", msg.content); + }); + + t.test("partial input keeps trailing incomplete sequence out", [&](testing & t) { + std::string input("<think>x</think>a\x80" "b\xE4\xB8"); + common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT); + auto result = parser.parse(ctx); + t.assert_true("not fail", !result.fail()); + + common_chat_msg msg; + auto mapper = common_chat_peg_mapper(msg); + mapper.from_ast(ctx.ast, result); + + t.assert_equal("content", "a\xEF\xBF\xBD" "b", msg.content); + }); +} diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 0cc7e00340a7..b88071c774c8 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -472,6 +472,12 @@ static common_chat_tool empty_args_tool_no_properties{ })", }; +static common_chat_tool empty_args_tool_no_schema{ + /* .name = */ "empty_args_no_schema", + /* .description = */ "A tool that takes no arguments and has no parameters schema", + /* .parameters = */ "{}", +}; + static common_chat_tool python_tool{ /* .name = */ "python", /* .description = */ "an ipython interpreter", @@ -840,6 +846,25 @@ static common_chat_tool nullable_int_tool{ })", }; +static common_chat_tool string_union_tool{ + /* .name = */ "set_union", + /* .description = */ "Set values whose types are unions with string", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "value": { + "type": ["string", "object"], + "description": "A string or object value" + }, + "amount": { + "type": ["string", "integer"], + "description": "A string or integer value" + } + }, + "required": ["value", "amount"] + })", +}; + static common_chat_tool enum_no_type_tool{ /* .name = */ "set_unit", /* .description = */ "Set a temperature unit", @@ -3007,6 +3032,26 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect(message_with_content_and_tool_call("Hello, world!\nWhat's up?", "get_time", R"({"city": "Paris"})")) .run(); + // Required tool call + tst.test( + "<|tool_call>call:get_time{city:<|\"|>Paris<|\"|>}<tool_call|>") + .tools({ get_time_tool }) + .tool_choice(COMMON_CHAT_TOOL_CHOICE_REQUIRED) + .expect(message_with_tool_calls("get_time", R"({"city": "Paris"})")) + .run(); + + // Required tool call after reasoning + tst.test( + "<|channel>thought\nI'm\nthinking<channel|><|tool_call>call:get_time{city:<|\"|>Paris<|\"|>}<tool_call|>") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ get_time_tool }) + .tool_choice(COMMON_CHAT_TOOL_CHOICE_REQUIRED) + .expect_reasoning("I'm\nthinking") + .expect_tool_calls({ + { "get_time", R"({"city": "Paris"})", {} }, + }) + .run(); + // Parallel tool calls tst.test( "<|tool_call>call:get_time{city:<|\"|>London<|\"|>}<tool_call|>" @@ -3799,6 +3844,46 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); + // nullable string given null - parses as JSON null, not the string "null" + tst.test( + "<tool_call>\n" + "<function=set_nullable_str>\n" + "<parameter=name>\nnull\n</parameter>\n" + "</function>\n" + "</tool_call>") + .tools({ nullable_string_tool }) + .expect_tool_calls({ + { "set_nullable_str", R"({"name": null})", {} }, + }) + .run(); + + // unions with string - JSON values of the other types are typed, everything else is a string + tst.test( + "<tool_call>\n" + "<function=set_union>\n" + "<parameter=value>\n{\"a\": 1}\n</parameter>\n" + "<parameter=amount>\n2 dollars\n</parameter>\n" + "</function>\n" + "</tool_call>") + .tools({ string_union_tool }) + .expect_tool_calls({ + { "set_union", R"({"value": {"a": 1}, "amount": "2 dollars"})", {} }, + }) + .run(); + + tst.test( + "<tool_call>\n" + "<function=set_union>\n" + "<parameter=value>\n{not valid json\n</parameter>\n" + "<parameter=amount>\n42\n</parameter>\n" + "</function>\n" + "</tool_call>") + .tools({ string_union_tool }) + .expect_tool_calls({ + { "set_union", R"({"value": "{not valid json", "amount": 42})", {} }, + }) + .run(); + // enum without explicit type key - should infer string from enum values tst.test( "<tool_call>\n" @@ -4504,6 +4589,100 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .run(); } + // Spark2.5 uses tagged arguments with forced-open thinking. + { + auto tst = peg_tester("models/templates/Spark2.5.jinja", detailed_debug); + + tst.test("Hello, world!\nWhat's up?") + .enable_thinking(false) + .expect(message_assist) + .expect_reconstruction() + .run(); + + tst.test("I'm\nthinking</think>Hello, world!\nWhat's up?") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .expect(message_assist_thoughts) + .expect_reconstruction() + .run(); + + tst.test( + "<tool_call>special_function" + "<arg_key>arg1</arg_key><arg_value>1</arg_value>" + "</tool_call>") + .enable_thinking(false) + .tools({ special_function_tool }) + .expect(message_assist_call) + .expect_reconstruction() + .run(); + + tst.test( + "I'm\nthinking</think>" + "<tool_call>special_function" + "<arg_key>arg1</arg_key><arg_value>1</arg_value>" + "</tool_call>") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ special_function_tool }) + .expect(message_assist_call_thoughts) + .expect_reconstruction() + .run(); + + tst.test( + "<tool_call>special_function" + "<arg_key>arg1</arg_key><arg_value>1</arg_value>" + "</tool_call>" + "<tool_call>special_function_with_opt" + "<arg_key>arg1</arg_key><arg_value>1</arg_value>" + "<arg_key>arg2</arg_key><arg_value>2</arg_value>" + "</tool_call>") + .enable_thinking(false) + .parallel_tool_calls(true) + .tools({ special_function_tool, special_function_tool_with_optional_param }) + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} }, + }) + .expect_reconstruction() + .run(); + + tst.test( + "Preparing updates." + "<tool_call>magic_int" + "<arg_key>ref</arg_key><arg_value>42</arg_value>" + "<arg_key>name</arg_key><arg_value>上海</arg_value>" + "</tool_call>" + "<tool_call>amount" + "<arg_key>orig</arg_key><arg_value>2.5</arg_value>" + "</tool_call>" + "<tool_call>toggle" + "<arg_key>enabled</arg_key><arg_value>true</arg_value>" + "</tool_call>" + "<tool_call>set_config" + "<arg_key>config</arg_key><arg_value>{\"source\": \"spark\", \"options\": {\"strict\": true}}</arg_value>" + "</tool_call>" + "<tool_call>nested_args" + "<arg_key>tags</arg_key><arg_value>[\"alpha\", \"测试\"]</arg_value>" + "<arg_key>entries</arg_key><arg_value>[{\"id\": 1, \"label\": \"first\"}, {\"id\": 2, \"label\": \"第二\"}]</arg_value>" + "</tool_call>" + "<tool_call>empty_args" + "</tool_call>") + .enable_thinking(false) + .parallel_tool_calls(true) + .tools({ magic_int_tool, amount_tool, toggle_tool, config_tool, nested_args_tool, empty_args_tool }) + .expect_content("Preparing updates.") + .expect_tool_calls({ + { "magic_int", R"({"ref": 42, "name": "上海"})", {} }, + { "amount", R"({"orig": 2.5})", {} }, + { "toggle", R"({"enabled": true})", {} }, + { "set_config", R"({"config": {"source": "spark", "options": {"strict": true}}})", {} }, + { "nested_args", R"({"tags": ["alpha", "测试"], "entries": [{"id": 1, "label": "first"}, {"id": 2, "label": "第二"}]})", {} }, + { "empty_args", "{}", {} }, + }) + .expect_reconstruction() + .run(); + } + // Verify the throw path produces a readable error message, not std::out_of_range. // #20424 introduced effective_input = generation_prompt + input, but the throw // uses input.substr(result.end) where result.end is in effective_input space. @@ -4561,6 +4740,214 @@ static void test_template_output_peg_parsers(bool detailed_debug) { } } + // Ling 3.0 / Bailing V3 dedicated parser + { + auto tst = peg_tester("models/templates/inclusionai-ling-3.0-flash.jinja", detailed_debug); + + const std::string get_time_call = + "<tool_call>get_time\n" + "<arg_key>city</arg_key>\n" + "<arg_value>Paris</arg_value>\n" + "</tool_call>"; + + // A tool call emitted before the think block is closed must be extracted, + // with the preceding text kept as reasoning. + tst.test("I need to check the time first.\n" + get_time_call) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ get_time_tool }) + .expect_reasoning("I need to check the time first.\n") + .expect_tool_calls({ { "get_time", R"({"city": "Paris"})", "" } }) + .run(); + + // Closed think block, prose, then a tool call. + tst.test("Let me check the time.\n</think>\nChecking it now.\n" + get_time_call) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ get_time_tool }) + .expect_reasoning("Let me check the time.\n") + .expect_content("Checking it now.\n") + .expect_tool_calls({ { "get_time", R"({"city": "Paris"})", "" } }) + .run(); + + // Prose after the last tool call is content, not a parse failure. + tst.test(get_time_call + "\nThe time has been checked.") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ get_time_tool }) + .expect_content("\nThe time has been checked.") + .expect_tool_calls({ { "get_time", R"({"city": "Paris"})", "" } }) + .run(); + + // Parallel tool calls. + tst.test("</think>\n" + get_time_call + "\n" + get_time_call) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ get_time_tool }) + .parallel_tool_calls(true) + .expect_content("") + .expect_tool_calls({ + { "get_time", R"({"city": "Paris"})", "" }, + { "get_time", R"({"city": "Paris"})", "" }, + }) + .run(); + + // Argument values may contain marker-like strings. + tst.test("check this\n</think>\n<tool_call>tool_2req_4opt\n" + "<arg_key>req1</arg_key>\n<arg_value>contains </think> and <tool_call> strings</arg_value>\n" + "<arg_key>req2</arg_key>\n<arg_value>1</arg_value>\n" + "</tool_call>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ tool_2req_4opt }) + .expect_reasoning("check this\n") + .expect_tool_calls({ + { "tool_2req_4opt", R"({"req1": "contains </think> and <tool_call> strings", "req2": 1})", "" }, + }) + .run(); + + // reasoning_format=none keeps extracting tool calls. + tst.test("I need to check the time first.\n" + get_time_call) + .reasoning_format(COMMON_REASONING_FORMAT_NONE) + .tools({ get_time_tool }) + .expect_content("I need to check the time first.\n") + .expect_tool_calls({ { "get_time", R"({"city": "Paris"})", "" } }) + .run(); + + // With thinking off the template pre-closes the think block, so the model + // emits bare content: it must not be classified as reasoning. + tst.test("Here is the answer.\nNo think block at all.") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .enable_thinking(false) + .expect_reasoning("") + .expect_content("Here is the answer.\nNo think block at all.") + .run(); + + tst.test(get_time_call) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .enable_thinking(false) + .tools({ get_time_tool }) + .expect_reasoning("") + .expect_tool_calls({ { "get_time", R"({"city": "Paris"})", "" } }) + .run(); + + // The end-of-turn token may arrive spelled out as text tokens instead of + // the single control token; it must not leak into content. + tst.test("Here is the answer.<|role_end|>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .enable_thinking(false) + .expect_content("Here is the answer.") + .run(); + + tst.test(get_time_call + "<|role_end|>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ get_time_tool }) + .expect_tool_calls({ { "get_time", R"({"city": "Paris"})", "" } }) + .run(); + + // Real output tolerates whitespace variation between tags (the template + // renders historical calls with no newline after the tool name). + tst.test("</think>\n<tool_call>get_time<arg_key>city</arg_key><arg_value>Paris</arg_value></tool_call>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ get_time_tool }) + .expect_tool_calls({ { "get_time", R"({"city": "Paris"})", "" } }) + .run(); + + // Required arguments may arrive in any order. + tst.test("</think>\n<tool_call>tool_2req_4opt\n" + "<arg_key>req2</arg_key>\n<arg_value>7</arg_value>\n" + "<arg_key>req1</arg_key>\n<arg_value>hello</arg_value>\n" + "</tool_call>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ tool_2req_4opt }) + .expect_tool_calls({ { "tool_2req_4opt", R"({"req2": 7, "req1": "hello"})", "" } }) + .run(); + + // Optional arguments may follow the required ones. + tst.test("</think>\n<tool_call>tool_2req_4opt\n" + "<arg_key>req1</arg_key>\n<arg_value>hello</arg_value>\n" + "<arg_key>req2</arg_key>\n<arg_value>7</arg_value>\n" + "<arg_key>opt1</arg_key>\n<arg_value>extra</arg_value>\n" + "</tool_call>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ tool_2req_4opt }) + .expect_tool_calls({ { "tool_2req_4opt", R"({"req1": "hello", "req2": 7, "opt1": "extra"})", "" } }) + .run(); + + // Non-string arguments parse as JSON. + tst.test("</think>\n<tool_call>magic_int\n" + "<arg_key>ref</arg_key>\n<arg_value>42</arg_value>\n" + "</tool_call>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ magic_int_tool }) + .expect_tool_calls({ { "magic_int", R"({"ref": 42})", "" } }) + .run(); + + // A nullable string accepts a JSON null and raw text. + tst.test("</think>\n<tool_call>set_nullable_str\n" + "<arg_key>name</arg_key>\n<arg_value>null</arg_value>\n" + "</tool_call>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ nullable_string_tool }) + .expect_tool_calls({ { "set_nullable_str", R"({"name": null})", "" } }) + .run(); + + tst.test("</think>\n<tool_call>set_nullable_str\n" + "<arg_key>name</arg_key>\n<arg_value>hello world</arg_value>\n" + "</tool_call>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ nullable_string_tool }) + .expect_tool_calls({ { "set_nullable_str", R"({"name": "hello world"})", "" } }) + .run(); + + // A raw string that starts like a JSON value must not be taken as JSON: + // the choice falls back to the string alternative. + tst.test("</think>\n<tool_call>set_nullable_str\n" + "<arg_key>name</arg_key>\n<arg_value>123 Main St</arg_value>\n" + "</tool_call>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ nullable_string_tool }) + .expect_tool_calls({ { "set_nullable_str", R"({"name": "123 Main St"})", "" } }) + .run(); + + // String unions: object and integer values parse as JSON, strings stay raw. + tst.test("</think>\n<tool_call>set_union\n" + "<arg_key>value</arg_key>\n<arg_value>{\"a\": 1}</arg_value>\n" + "<arg_key>amount</arg_key>\n<arg_value>7</arg_value>\n" + "</tool_call>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ string_union_tool }) + .expect_tool_calls({ { "set_union", R"({"value": {"a": 1}, "amount": 7})", "" } }) + .run(); + + tst.test("</think>\n<tool_call>set_union\n" + "<arg_key>value</arg_key>\n<arg_value>plain text</arg_value>\n" + "<arg_key>amount</arg_key>\n<arg_value>1abc</arg_value>\n" + "</tool_call>") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ string_union_tool }) + .expect_tool_calls({ { "set_union", R"({"value": "plain text", "amount": "1abc"})", "" } }) + .run(); + + // Continuation: the partial assistant turn is spliced back into the prompt. + common_chat_msg prefill = simple_assist_msg("", "I'm thinking"); + + tst.test("Hello, world!") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .enable_thinking(true) + .messages({ message_user, prefill }) + .add_generation_prompt(false) + .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT) + .expect_reasoning("I'm thinking") + .expect_content("Hello, world!") + .run(); + + tst.test(" more</think>Hello, world!") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .enable_thinking(true) + .messages({ message_user, prefill }) + .add_generation_prompt(false) + .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING) + .expect_reasoning("I'm thinking more") + .expect_content("Hello, world!") + .run(); + } + // Kimi-K3 tests - custom parser // Unique feature: XTML tags built from <|open|>/<|close|>/<|sep|>, and a // generation prompt that leaves the think section already open. @@ -5094,6 +5481,13 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect(simple_assist_msg("", "", "empty_args", "{}")) .run(); + // Tool call with no parameters schema, {} means no arguments + tst.test("<tool_call>\n{\"name\": \"empty_args_no_schema\", \"arguments\": {}}</tool_call>") + .enable_thinking(false) + .tools({ empty_args_tool_no_schema }) + .expect(simple_assist_msg("", "", "empty_args_no_schema", "{}")) + .run(); + // fake tool call marker in reasoning tst.test( "Let me think about <tool_call>\n{\"name\": \"special_function\", \"arguments\": {\"arg1\": 2}}</tool_call> hmm\n</think>\n\n" @@ -6080,6 +6474,14 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect(message_assist) .run(); + // A tool call as the first message of the turn: "<|start|>assistant" is the + // generation prompt, so the output starts at " to=". + tst.test(" to=special_function<|message|>" + call_markup) + .tools({ special_function_tool }) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .expect(message_assist_call) + .run(); + // "Inform then act": the model answers the user and calls a tool in ONE generation, // closing the answer with <|eom|>. The answer must stop there rather than swallow it. tst.test(" to=user<|message|>Hello, world!\nWhat's up?<|eom|>" diff --git a/tests/test-fusion.cpp b/tests/test-fusion.cpp new file mode 100644 index 000000000000..467248f0f101 --- /dev/null +++ b/tests/test-fusion.cpp @@ -0,0 +1,565 @@ +// test-fusion: verify the backend fusion logic against a per-device baseline. +// +// for every dummy model generated by test-llama-archs, the tool runs the model on a single +// device with fusion enabled and disabled, and reports: +// - the per-fusion-type counters for each mode (prefill / decode, merged into "any" when the +// per-graph counts match) +// - the NMSE between the fused and unfused logits +// - the NMSE between the device and a CPU reference +// +// the per-fusion-type counters are compared against a per-device baseline file (CSV) so a +// fusion pattern that silently stops matching (or fires when it should not) is caught as a +// regression. +// +// usage: +// test-fusion --models DIR --device MTL0 --record baseline.csv # generate a baseline +// test-fusion --models DIR --device MTL0 --check baseline.csv # validate against it +// test-fusion --model FILE --device MTL0 --check baseline.csv # validate a single model + +#include "common.h" +#include "log.h" +#include "llama-cpp.h" + +#include "ggml.h" +#include "gguf.h" + +#include <algorithm> +#include <array> +#include <cstring> +#include <filesystem> +#include <fstream> +#include <iomanip> +#include <iostream> +#include <map> +#include <random> +#include <string> +#include <vector> + +// generic fusion debugging API, resolved through the ad-hoc get_proc_address mechanism +// (not part of the official ggml backend interface yet). a backend that adopts fusion debugging +// exports these exact names. +typedef void * ggml_backend_fusion_t; + +typedef ggml_backend_fusion_t ( * fusion_get_t) (ggml_backend_dev_t); +typedef void ( * fusion_stats_init_t) (ggml_backend_fusion_t); +typedef void ( * fusion_stats_reset_t) (ggml_backend_fusion_t); +typedef int ( * fusion_stats_get_t) (ggml_backend_fusion_t, const char **, uint64_t *, int); +typedef void ( * fusion_set_enabled_t) (ggml_backend_fusion_t, bool); + +static bool silent_model_load_progress(float, void *) { + return true; +} + +struct gguf_context_ptr { + gguf_context * ctx; + gguf_context_ptr(gguf_context * c) : ctx(c) {} + ~gguf_context_ptr() { if (ctx) { gguf_free(ctx); } } + gguf_context * get() const { return ctx; } + gguf_context_ptr(const gguf_context_ptr &) = delete; + gguf_context_ptr & operator=(const gguf_context_ptr &) = delete; +}; + +// NMSE between two vectors (same as tests/test-llama-archs.cpp) +static double nmse(const std::vector<float> & a, const std::vector<float> & b) { + GGML_ASSERT(a.size() == b.size()); + double mse_a_b = 0.0; + double mse_a_0 = 0.0; + + for (size_t i = 0; i < a.size(); i++) { + const float a_i = a[i]; + const float b_i = b[i]; + + mse_a_b += (a_i - b_i) * (a_i - b_i); + mse_a_0 += a_i * a_i; + } + + return mse_a_b / mse_a_0; +} + +// deterministic token sequence +static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed) { + std::mt19937 gen(seed); + std::uniform_int_distribution<> dis(0, n_vocab - 1); + std::vector<llama_token> ret; + ret.reserve(n_tokens); + for (uint32_t i = 0; i < n_tokens; i++) { + ret.push_back(dis(gen)); + } + return ret; +} + +// trim leading/trailing whitespace (used when parsing padded CSV columns) +static std::string trim(const std::string & s) { + const size_t b = s.find_first_not_of(" \t\r\n"); + if (b == std::string::npos) { + return ""; + } + const size_t e = s.find_last_not_of(" \t\r\n"); + return s.substr(b, e - b + 1); +} + +static std::string get_arch(const std::string & path) { + gguf_init_params params = { /*no_alloc=*/true, /*ctx=*/nullptr }; + gguf_context_ptr ctx(gguf_init_from_file(path.c_str(), params)); + if (!ctx.get()) { + throw std::runtime_error("failed to read gguf: " + path); + } + const int idx = gguf_find_key(ctx.get(), "general.architecture"); + if (idx < 0) { + return "unknown"; + } + const char * val = gguf_get_val_str(ctx.get(), idx); + return val ? val : "unknown"; +} + +static llama_model_ptr load_model(const std::string & path, ggml_backend_dev_t dev) { + llama_model_params model_params = llama_model_default_params(); + model_params.progress_callback = silent_model_load_progress; + std::vector<ggml_backend_dev_t> devs = { dev, nullptr }; + model_params.devices = devs.data(); + model_params.split_mode = LLAMA_SPLIT_MODE_LAYER; + + llama_model_ptr model(llama_model_load_from_file(path.c_str(), model_params)); + if (!model) { + throw std::runtime_error("failed to load model: " + path); + } + return model; +} + +// a fresh context (fresh state) from an already-loaded model +static llama_context_ptr create_ctx(llama_model * model, int n_ubatch) { + llama_context_params ctx_params = llama_context_default_params(); + ctx_params.n_ctx = 0; + ctx_params.n_threads = 4; + ctx_params.n_threads_batch = 4; + ctx_params.n_ubatch = n_ubatch; + ctx_params.n_batch = n_ubatch; + + llama_context_ptr lctx(llama_init_from_model(model, ctx_params)); + if (!lctx) { + throw std::runtime_error("failed to init context"); + } + return lctx; +} + +// decode all tokens in one batch; returns the logits of every token +static std::vector<float> decode_prefill(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) { + const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + llama_batch batch = llama_batch_init(tokens.size(), 0, 1); + for (size_t i = 0; i < tokens.size(); i++) { + common_batch_add(batch, tokens[i], i, { 0 }, true); + } + batch.n_tokens = tokens.size(); + if (llama_decode(lctx, batch)) { + llama_batch_free(batch); + throw std::runtime_error("prefill decode failed"); + } + + std::vector<float> ret; + ret.reserve(tokens.size() * n_vocab); + for (size_t i = 0; i < tokens.size(); i++) { + const float * logits_ith = llama_get_logits_ith(lctx, i); + for (uint32_t j = 0; j < n_vocab; j++) { + ret.push_back(logits_ith[j]); + } + } + llama_batch_free(batch); + return ret; +} + +// decode one token at a time; returns the logits of the last token of each step +static std::vector<float> decode_gen(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) { + const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + llama_batch batch = llama_batch_init(1, 0, 1); + std::vector<float> ret; + for (size_t i = 0; i < tokens.size(); i++) { + common_batch_clear(batch); + common_batch_add(batch, tokens[i], i, { 0 }, true); + if (llama_decode(lctx, batch)) { + llama_batch_free(batch); + throw std::runtime_error("decode failed"); + } + const float * logits = llama_get_logits_ith(lctx, 0); + for (uint32_t j = 0; j < n_vocab; j++) { + ret.push_back(logits[j]); + } + } + llama_batch_free(batch); + return ret; +} + +static void read_counts(fusion_stats_get_t api_stats_get, ggml_backend_fusion_t finfo, + std::vector<const char *> & labels, std::vector<uint64_t> & counts) { + const int n = api_stats_get(finfo, nullptr, nullptr, 0); + labels.assign(n, nullptr); + counts.assign(n, 0); + api_stats_get(finfo, labels.data(), counts.data(), n); +} + +// one row of the per-label report +struct fusion_row { + std::string arch; + bool moe; + std::string mode; + std::string label; + uint64_t count_fused; + uint64_t count_unfused; + uint64_t expected; + double nmse_fus; + double nmse_dev; + bool ok_count; // counts match the baseline + bool ok_nmse; // nmse within epsilon +}; + +static void usage(const char * argv0) { + printf("%s: verify fusion counts on a device against a per-device baseline\n\n", argv0); + printf("usage: %s [options]\n\n", argv0); + printf("options:\n"); + printf(" --models DIR run over all .gguf models in a directory\n"); + printf(" --model FILE run over a single model file (mutually exclusive with --models)\n"); + printf(" --device NAME device to run on (e.g. MTL0, CPU)\n"); + printf(" --record CSV write the golden baseline\n"); + printf(" --check CSV validate the counters against a baseline (default)\n"); + printf(" -h, --help show this message and exit\n"); +} + +int main(int argc, char ** argv) { + std::string models_dir; + std::string model_file; + std::string device_name; + std::string record_path; + std::string check_path; + + for (int i = 1; i < argc; i++) { + const std::string arg = argv[i]; + const auto next = [&](const char * name) -> std::string { + if (i + 1 >= argc) { + LOG_ERR("%s: %s requires an argument\n", __func__, name); + exit(1); + } + return argv[++i]; + }; + if (arg == "-h" || arg == "--help") { + usage(argv[0]); + exit(0); + } + if (arg == "--models") { models_dir = next("--models"); } + else if (arg == "--model") { model_file = next("--model"); } + else if (arg == "--device"){ device_name = next("--device"); } + else if (arg == "--record"){ record_path = next("--record"); } + else if (arg == "--check") { check_path = next("--check"); } + else { + LOG_ERR("%s: unknown argument: %s\n", __func__, arg.c_str()); + return 1; + } + } + + if (device_name.empty()) { + LOG_ERR("%s: --device NAME is required\n", __func__); + return 1; + } + if (models_dir.empty() && model_file.empty()) { + LOG_ERR("%s: --models DIR or --model FILE is required\n", __func__); + return 1; + } + if (!models_dir.empty() && !model_file.empty()) { + LOG_ERR("%s: --models DIR and --model FILE are mutually exclusive\n", __func__); + return 1; + } + if (!record_path.empty() && !check_path.empty()) { + LOG_ERR("%s: --record and --check are mutually exclusive\n", __func__); + return 1; + } + + std::vector<std::string> models; + if (!model_file.empty()) { + if (!std::filesystem::is_regular_file(model_file)) { + LOG_ERR("%s: model file '%s' does not exist\n", __func__, model_file.c_str()); + return 1; + } + models.push_back(model_file); + } else { + if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) { + LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str()); + return 1; + } + for (const auto & entry : std::filesystem::directory_iterator(models_dir)) { + if (entry.is_regular_file() && entry.path().extension() == ".gguf") { + models.push_back(entry.path().string()); + } + } + std::sort(models.begin(), models.end()); + + if (models.empty()) { + LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str()); + return 1; + } + } + + common_init(); + ggml_backend_load_all(); + + ggml_backend_dev_t dev = ggml_backend_dev_by_name(device_name.c_str()); + if (!dev) { + LOG_WRN("%s: device '%s' not found - skipping (baseline is device-specific)\n", + __func__, device_name.c_str()); + return 0; + } + + // resolve the generic fusion debugging functions through the ad-hoc get_proc_address + // mechanism; a backend that does not adopt fusion debugging exports none of them + auto * reg = ggml_backend_dev_backend_reg(dev); + + // output naming uses the backend base name (e.g. "MTL") rather than the specific device + // name (e.g. "MTL0") the test was invoked with + const std::string base_name = ggml_backend_reg_name(reg); + + auto api_get = (fusion_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_get"); + auto api_stats_init = (fusion_stats_init_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_init"); + auto api_stats_reset = (fusion_stats_reset_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_reset"); + auto api_stats_get = (fusion_stats_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_get"); + auto api_set_enabled = (fusion_set_enabled_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_set_enabled"); + + if (!api_get || !api_stats_init || !api_set_enabled || !api_stats_reset || !api_stats_get) { + LOG_ERR("%s: device '%s' does not export the generic fusion debugging API " + "(ggml_backend_fusion_*) - cannot run the fusion regression test\n", + __func__, device_name.c_str()); + return 1; + } + + ggml_backend_fusion_t finfo = api_get(dev); + + // enable fusions stats + api_stats_init(finfo); + + const bool has_counts = true; + + // load the baseline (if any): key arch|moe|mode|label -> expected count + std::map<std::string, uint64_t> baseline; + if (!check_path.empty()) { + std::ifstream in(check_path); + if (!in) { + LOG_ERR("%s: cannot open baseline '%s'\n", __func__, check_path.c_str()); + return 1; + } + std::string line; + while (std::getline(in, line)) { + if (line.empty() || line[0] == '#') { + continue; + } + std::vector<std::string> cols; + size_t pos = 0; + while ((pos = line.find(',')) != std::string::npos) { + cols.push_back(trim(line.substr(0, pos))); + line.erase(0, pos + 1); + } + cols.push_back(trim(line)); + if (cols.size() != 5) { + continue; + } + baseline[cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3]] = std::stoull(cols[4]); + } + } + + std::vector<fusion_row> rows; + + LOG_INF("%s: running fusion test over %zu models on '%s'\n", __func__, models.size(), base_name.c_str()); + + const size_t seed = 1; + + for (const auto & model_path : models) { + const std::string arch = get_arch(model_path); + const bool moe = arch.find("moe") != std::string::npos; + + llama_model_ptr model; + llama_model_ptr model_cpu; + uint32_t n_vocab = 0; + try { + model = load_model(model_path, dev); + model_cpu = load_model(model_path, ggml_backend_dev_by_name("CPU")); + n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model.get())); + } catch (const std::exception & e) { + LOG_ERR("%s: %s: %s\n", __func__, model_path.c_str(), e.what()); + continue; + } + + struct mode_cfg { + std::string name; + std::vector<float> (*decode)(llama_model *, llama_context *, const std::vector<llama_token> &); + int n_tokens; + int n_graphs; // graph runs per mode (prefill=1, decode=16) + }; + const mode_cfg modes[] = { + { "prefill", decode_prefill, 32, 1 }, + { "decode", decode_gen, 16, 16 }, + }; + + // per-label, per-mode data for this model; prefill and decode are merged into a single + // "any" row when their per-graph counts match + struct mode_data { + bool present; + uint64_t count_fused; // per graph + uint64_t count_unfused; // per graph + double nmse_fus; + double nmse_dev; + bool ok_nmse; + }; + std::map<std::string, std::array<mode_data, 2>> mdata; + + for (int mi = 0; mi < 2; mi++) { + const mode_cfg & mode = modes[mi]; + const auto tokens = get_tokens(mode.n_tokens, n_vocab, seed); + + // CPU reference for this mode (fresh context, fresh state) + std::vector<float> logits_cpu; + try { + llama_context_ptr ctx = create_ctx(model_cpu.get(), 32); + logits_cpu = mode.decode(model_cpu.get(), ctx.get(), tokens); + } catch (const std::exception & e) { + LOG_WRN("%s: %s: cpu reference: %s\n", __func__, model_path.c_str(), e.what()); + } + + // fused run on a fresh context (fresh state) + std::vector<float> logits_fused; + std::vector<const char *> labels; + std::vector<uint64_t> counts_fused; + { + llama_context_ptr ctx = create_ctx(model.get(), 32); + if (has_counts) { + api_set_enabled(finfo, true); + api_stats_reset(finfo); + } + logits_fused = mode.decode(model.get(), ctx.get(), tokens); + if (has_counts) { + read_counts(api_stats_get, finfo, labels, counts_fused); + } + } + + // unfused run on another fresh context (fresh state) + std::vector<float> logits_unfused; + std::vector<uint64_t> counts_unfused; + { + llama_context_ptr ctx = create_ctx(model.get(), 32); + if (has_counts) { + api_set_enabled(finfo, false); + api_stats_reset(finfo); + } + logits_unfused = mode.decode(model.get(), ctx.get(), tokens); + if (has_counts) { + read_counts(api_stats_get, finfo, labels, counts_unfused); + } + } + + const double nmse_fus = nmse(logits_fused, logits_unfused); + const double nmse_dev = logits_cpu.empty() ? 0.0 : nmse(logits_fused, logits_cpu); + + if (has_counts) { + for (int i = 0; i < (int) labels.size(); i++) { + const uint64_t fused = counts_fused[i] / mode.n_graphs; + const uint64_t unfused = counts_unfused[i] / mode.n_graphs; + if (fused == 0 && unfused == 0) { + continue; + } + auto & d = mdata[labels[i]][mi]; + d.present = true; + d.count_fused = fused; + d.count_unfused = unfused; + d.nmse_fus = nmse_fus; + d.nmse_dev = nmse_dev; + d.ok_nmse = nmse_fus <= 1e-4; + } + } else { + rows.push_back({ arch, moe, mode.name, "?", 0, 0, 0, nmse_fus, nmse_dev, true, nmse_fus <= 1e-4 }); + } + } + + // build the per-label rows, merging prefill and decode into "any" when the per-graph + // counts match (they always do for the deterministic fusion table) + if (has_counts) { + for (auto & kv : mdata) { + const std::string & label = kv.first; + const auto & d = kv.second; + const bool both = d[0].present && d[1].present; + const bool match = both && d[0].count_fused == d[1].count_fused; + + if (match) { + // one "any" row; use the worst NMSE across the two modes + const std::string any_key = arch + "|" + (moe ? "1" : "0") + "|any|" + label; + const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key) : 0; + const bool ok_count = check_path.empty() || d[0].count_fused == expected; + const bool ok_nmse = d[0].ok_nmse && d[1].ok_nmse; + const double nmse_fus = std::max(d[0].nmse_fus, d[1].nmse_fus); + const double nmse_dev = std::max(d[0].nmse_dev, d[1].nmse_dev); + rows.push_back({ arch, moe, "any", label, d[0].count_fused, d[0].count_unfused, + expected, nmse_fus, nmse_dev, ok_count, ok_nmse }); + } else { + // counts differ - keep a separate row per mode + for (int mi = 0; mi < 2; mi++) { + if (!d[mi].present) { + continue; + } + const mode_data & a = d[mi]; + const std::string mode_key = arch + "|" + (moe ? "1" : "0") + "|" + modes[mi].name + "|" + label; + const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key) : 0; + const bool ok_count = check_path.empty() || a.count_fused == expected; + rows.push_back({ arch, moe, modes[mi].name, label, a.count_fused, a.count_unfused, + expected, a.nmse_fus, a.nmse_dev, ok_count, a.ok_nmse }); + } + } + } + } + + LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str()); + } + + // print the report + { + std::ofstream out(record_path); + std::ostream & os = record_path.empty() ? std::cout : out; + if (!record_path.empty()) { + os << "# test-fusion baseline for device " << base_name << "\n"; + os << "# " << std::left + << std::setw(18) << "arch" << ',' + << std::setw(4) << "moe" << ',' + << std::setw(8) << "mode" << ',' + << std::setw(28) << "label" << ',' + << std::right << std::setw(7) << "count" << '\n'; + } + + LOG_INF("%-20s %-4s %-8s %-22s %7s %7s %7s %10s %10s %s\n", + "arch", "moe", "mode", "label", "fused", "unfused", "expected", "nmse_fus", "nmse_dev", "status"); + int n_ok = 0; + int n_bad = 0; + for (const auto & r : rows) { + const bool ok = r.ok_count && r.ok_nmse; + const char * status = ok ? "ok" : "FAIL"; + if (ok) { n_ok++; } else { n_bad++; } + LOG_INF("%-20s %-4s %-8s %-22s %7llu %7llu %7llu %10.2e %10.2e %s\n", + r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label.c_str(), + (unsigned long long) r.count_fused, (unsigned long long) r.count_unfused, + (unsigned long long) r.expected, r.nmse_fus, r.nmse_dev, status); + if (!record_path.empty()) { + os << std::left + << std::setw(20) << r.arch << ',' + << std::setw(4) << (r.moe ? "1" : "0") << ',' + << std::setw(8) << r.mode << ',' + << std::setw(28) << r.label << ',' + << std::right << std::setw(7) << r.count_fused << '\n'; + } + } + LOG_INF("summary: %d ok, %d failed\n", n_ok, n_bad); + if (!record_path.empty()) { + LOG_INF("%s: baseline written to '%s'\n", __func__, record_path.c_str()); + } + + if (n_bad && !models_dir.empty() && !check_path.empty()) { + LOG_WRN("%s: if the fusion counts are expected to change, run with --record to update the baseline:\n" + "\n" + "./bin/test-llama-archs -o %s\n" + "%s --device %s --models %s --record %s\n", + __func__, models_dir.c_str(), argv[0], device_name.c_str(), models_dir.c_str(), check_path.c_str()); + } + + return n_bad; + } +} diff --git a/tests/test-gguf.cpp b/tests/test-gguf.cpp index fc636186f4c5..f40d6984bff9 100644 --- a/tests/test-gguf.cpp +++ b/tests/test-gguf.cpp @@ -1167,15 +1167,17 @@ static bool same_tensor_data(const struct ggml_context * orig, const struct ggml enum roundtrip_read_mode { ROUNDTRIP_READ_MODE_FILE, + ROUNDTRIP_READ_MODE_FILE_OFFSET, // GGUF embedded after some bytes of a bigger file ROUNDTRIP_READ_MODE_BUFFER, ROUNDTRIP_READ_MODE_CALLBACK, }; static const char * roundtrip_read_mode_name(const roundtrip_read_mode mode) { switch (mode) { - case ROUNDTRIP_READ_MODE_FILE: return "file"; - case ROUNDTRIP_READ_MODE_BUFFER: return "buffer"; - case ROUNDTRIP_READ_MODE_CALLBACK: return "callback"; + case ROUNDTRIP_READ_MODE_FILE: return "file"; + case ROUNDTRIP_READ_MODE_FILE_OFFSET: return "file_offset"; + case ROUNDTRIP_READ_MODE_BUFFER: return "buffer"; + case ROUNDTRIP_READ_MODE_CALLBACK: return "callback"; } GGML_ABORT("fatal error"); @@ -1214,6 +1216,12 @@ static std::pair<int, int> test_roundtrip( GGML_ASSERT(file); #endif // _WIN32 + // not a multiple of any alignment, so the data section padding must be relative to the GGUF start + const long prefix = read_mode == ROUNDTRIP_READ_MODE_FILE_OFFSET ? 7 : 0; + for (long i = 0; i < prefix; ++i) { + fputc(0xAB, file); + } + gguf_write_to_file_ptr(gguf_ctx_0, file, only_meta); rewind(file); @@ -1236,6 +1244,7 @@ static std::pair<int, int> test_roundtrip( }; gguf_ctx_1 = gguf_init_from_callback(read_buffer_callback, &reader, 4096, 4ull << 30 /* 4GB */, gguf_params); } else { + GGML_ASSERT(fseek(file, prefix, SEEK_SET) == 0); gguf_ctx_1 = gguf_init_from_file_ptr(file, gguf_params); } @@ -1451,6 +1460,11 @@ int main(int argc, char ** argv) { npass += result.first; ntest += result.second; } + { + std::pair<int, int> result = test_roundtrip(dev, seed, /*only_meta=*/false, ROUNDTRIP_READ_MODE_FILE_OFFSET); + npass += result.first; + ntest += result.second; + } { std::pair<int, int> result = test_roundtrip(dev, seed, /*only_meta=*/false, ROUNDTRIP_READ_MODE_BUFFER); npass += result.first; diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp index eb4b7c78f50f..2af96f8b4797 100644 --- a/tests/test-grammar-integration.cpp +++ b/tests/test-grammar-integration.cpp @@ -918,7 +918,7 @@ static void test_json_schema() { // Otherwise, this test structure is the same. test_schema( - "empty schema (object)", + "empty schema (any value)", // Schema R"""( {} @@ -927,14 +927,16 @@ static void test_json_schema() { { R"""({})""", R"""({"foo": "bar"})""", - }, - // Failing strings - { - "", "[]", "null", R"""("")""", "true", + }, + // Failing strings + { + "", + R"""({"foo"})""", + "foo", } ); diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index 974a3f9dd8df..cd8fa723ca02 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -374,12 +374,42 @@ static void test_expressions(testing & t) { "42" ); + test_template(t, "none in object", + "{{ x in {'low': 1, 'high': 2} }}", + {{"x", nullptr}}, + "False" + ); + + test_template(t, "none not in object", + "{{ x not in {'low': 1, 'high': 2} }}", + {{"x", nullptr}}, + "True" + ); + + test_template(t, "none in array", + "{{ x in [1, none, 3] }}", + {{"x", nullptr}}, + "True" + ); + test_template(t, "dot notation", "{{ user.name }}", {{"user", {{"name", "Bob"}}}}, "Bob" ); + test_template(t, "dot notation (integer property)", + "{{ {10: 'Bob'}.10 }}", + json::object(), + "Bob" + ); + + test_template(t, "dot notation (array index)", + "{{ user.10 }}", + {{"user", json::array({"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"})}}, + "k" + ); + test_template(t, "negative float (not dot notation)", "{{ -1.0 }}", json::object(), @@ -428,6 +458,49 @@ static void test_expressions(testing & t) { "['b']" ); + test_template(t, "array slice negative variable", + "{{ items[:-n]|string }}", + {{"items", json::array({"a", "b", "c"})}, {"n", 1}}, + "['a', 'b']" + ); + + test_template(t, "array slice negative variable indent", + "{{ indent[:-indent_factor] }}", + {{"indent", " "}, {"indent_factor", 2}}, + " " + ); + + test_template(t, "unary minus variable", + "{{ -n }}", + {{"n", 3}}, + "-3" + ); + + test_template(t, "unary plus variable", + "{{ +n }}", + {{"n", -3}}, + "-3" + ); + + test_template(t, "unary plus float", + "{{ +x }}", + {{"x", -1.5}}, + "-1.5" + ); + + // Unary binds tighter than filter: -n|abs == (-n)|abs, not -(n|abs) + test_template(t, "unary minus then abs filter", + "{{ -n|abs }}", + {{"n", -3}}, + "3" + ); + + test_template(t, "unary minus then number test", + "{{ -n is number }}", + {{"n", 3}}, + "True" + ); + test_template(t, "array slice step", "{{ items[::2]|string }}", {{"items", json::array({"a", "b", "c"})}}, diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index 2a1b6348c951..d5873c19d03e 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -9,8 +9,6 @@ #include "json.h" #include <cassert> -#include <fstream> -#include <sstream> #include <regex> static std::string trim(const std::string & source) { @@ -64,21 +62,8 @@ struct TestCase { } }; -static void write(const std::string & file, const std::string & content) { - std::ofstream f; - f.open(file.c_str()); - f << content.c_str(); - f.close(); -} - -static std::string read(const std::string & file) { - std::ostringstream actuals; - actuals << std::ifstream(file.c_str()).rdbuf(); - return actuals.str(); -} - -static void test_all(const std::string & lang, std::function<void(const TestCase &)> runner) { - fprintf(stderr, "#\n# Testing JSON schema conversion (%s)\n#\n", lang.c_str()); +static void test_all(const std::string & title, std::function<void(const TestCase &)> runner) { + fprintf(stderr, "#\n# %s\n#\n", title.c_str()); auto test = [&](const TestCase & tc) { fprintf(stderr, "- %s%s\n", tc.name.c_str(), tc.expected_status == FAILURE ? " (failure expected)" : ""); runner(tc); @@ -330,7 +315,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase test({ SUCCESS, - "empty schema (object)", + "empty schema (any value)", "{}", R"""( array ::= "[" space ( value ("," space value)* )? space "]" @@ -341,7 +326,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase null ::= "null" number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? space "}" - root ::= object + root ::= value space ::= | " " | "\n"{1,2} [ \t]{0,20} string ::= "\"" char* "\"" value ::= object | array | string | number | boolean | null @@ -569,6 +554,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase )""" }); + // items {} constrains nothing, the same as no items at all test({ SUCCESS, "array with empty items", @@ -582,11 +568,10 @@ static void test_all(const std::string & lang, std::function<void(const TestCase char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) decimal-part ::= [0-9]{1,16} integral-part ::= [0] | [1-9] [0-9]{0,15} - item ::= object null ::= "null" number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? space "}" - root ::= "[" space (item ("," space item)*)? space "]" + root ::= "[" space ( value ("," space value)* )? space "]" space ::= | " " | "\n"{1,2} [ \t]{0,20} string ::= "\"" char* "\"" value ::= object | array | string | number | boolean | null @@ -607,11 +592,10 @@ static void test_all(const std::string & lang, std::function<void(const TestCase char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) decimal-part ::= [0-9]{1,16} integral-part ::= [0] | [1-9] [0-9]{0,15} - item ::= object null ::= "null" number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? space "}" - root ::= "[" space (item ("," space item)*)? space "]" + root ::= "[" space ( value ("," space value)* )? space "]" space ::= | " " | "\n"{1,2} [ \t]{0,20} string ::= "\"" char* "\"" value ::= object | array | string | number | boolean | null @@ -1434,88 +1418,126 @@ static void test_all(const std::string & lang, std::function<void(const TestCase space ::= | " " | "\n"{1,2} [ \t]{0,20} )""" }); -} -static void test_resolves_to_string() { - fprintf(stderr, "#\n# Testing resolves_to_string\n#\n"); - - auto test = [](const std::string & name, const std::string & schema_str, bool expected) { - fprintf(stderr, "- %s\n", name.c_str()); - common_schema_info info; - auto schema = common_json::parse(schema_str); - info.resolve_refs(schema); - bool result = info.resolves_to_string(schema); - if (result != expected) { - fprintf(stderr, "#\n# Test '%s' failed.\n#\n", name.c_str()); - fprintf(stderr, "Schema: %s\n", schema_str.c_str()); - fprintf(stderr, "Expected: %s, Got: %s\n", expected ? "true" : "false", result ? "true" : "false"); - assert(false); - } - }; + test({ + SUCCESS, + "regexp with non-capturing group", + R"""({ + "type": "string", + "pattern": "^(?:foo|bar)baz$" + })""", + R"""( + root ::= "\"" (("foo" | "bar") "baz") "\"" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); - // Basic type checks - test("type string", R"({"type": "string"})", true); - test("type integer", R"({"type": "integer"})", false); - test("type number", R"({"type": "number"})", false); - test("type boolean", R"({"type": "boolean"})", false); - test("type object", R"({"type": "object"})", false); - test("type array", R"({"type": "array"})", false); - - // Type array (nullable string) - test("type array with string", R"({"type": ["string", "null"]})", true); - test("type array without string", R"({"type": ["integer", "null"]})", false); - - // String-specific keywords - test("minLength implies string", R"({"minLength": 1})", true); - test("maxLength implies string", R"({"maxLength": 10})", true); - test("pattern implies string", R"({"pattern": "^[a-z]+$"})", true); - - // Format - test("format date", R"({"format": "date"})", true); - test("format uuid", R"({"format": "uuid"})", true); - test("format email", R"({"format": "email"})", true); - - // Const - test("const string", R"({"const": "hello"})", true); - test("const number", R"({"const": 123})", false); - - // Enum - test("enum with strings", R"({"enum": ["a", "b", "c"]})", true); - test("enum with numbers", R"({"enum": [1, 2, 3]})", false); - test("enum mixed with string", R"({"enum": [1, "a", null]})", true); - - // anyOf - test("anyOf with string", R"({"anyOf": [{"type": "string"}, {"type": "integer"}]})", true); - test("anyOf without string", R"({"anyOf": [{"type": "integer"}, {"type": "boolean"}]})", false); - - // oneOf - test("oneOf with string", R"({"oneOf": [{"type": "string"}, {"type": "number"}]})", true); - test("oneOf without string", R"({"oneOf": [{"type": "object"}, {"type": "array"}]})", false); - - // allOf - all must be strings - test("allOf all strings", R"({"allOf": [{"type": "string"}, {"minLength": 1}]})", true); - test("allOf mixed types", R"({"allOf": [{"type": "string"}, {"type": "integer"}]})", false); - - // $ref - test("$ref to string", - R"({"$ref": "#/$defs/str", "$defs": {"str": {"type": "string"}}})", true); - test("$ref to integer", - R"({"$ref": "#/$defs/num", "$defs": {"num": {"type": "integer"}}})", false); - - // Nested - test("nested anyOf with string", - R"({"anyOf": [{"anyOf": [{"type": "integer"}, {"type": "string"}]}, {"type": "boolean"}]})", true); - - fprintf(stderr, "All resolves_to_string tests passed!\n"); -} + test({ + SUCCESS, + "regexp with nested non-capturing groups", + R"""({ + "type": "string", + "pattern": "^(?:(?:ab)+c)?d$" + })""", + R"""( + root ::= "\"" ((("ab")+ "c")? "d") "\"" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); -int main() { - fprintf(stderr, "LLAMA_NODE_AVAILABLE = %s\n", getenv("LLAMA_NODE_AVAILABLE") ? "true" : "false"); - fprintf(stderr, "LLAMA_PYTHON_AVAILABLE = %s\n", getenv("LLAMA_PYTHON_AVAILABLE") ? "true" : "false"); + test({ + SUCCESS, + "unanchored regexp", + R"""({ + "type": "string", + "pattern": "[0-9]+" + })""", + R"""( + char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) + root ::= string + space ::= | " " | "\n"{1,2} [ \t]{0,20} + string ::= "\"" char* "\"" + )""" + }); - test_resolves_to_string(); + // the rules of the partial conversion (here "root-0") must not leak into the grammar + test({ + SUCCESS, + "regexp with unsupported shorthand", + R"""({ + "type": "string", + "pattern": "^[0-9]{3}\\w$" + })""", + R"""( + char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) + root ::= string + space ::= | " " | "\n"{1,2} [ \t]{0,20} + string ::= "\"" char* "\"" + )""" + }); - test_all("C++", [](const TestCase & tc) { + test({ + SUCCESS, + "regexp with escaped hyphen in a character class", + R"""({ + "type": "string", + "pattern": "^[a-z\\-]+$" + })""", + R"""( + root ::= "\"" ([a-z\-]+) "\"" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); + + test({ + SUCCESS, + "regexp with escaped hyphen outside a character class", + R"""({ + "type": "string", + "pattern": "^a\\-b$" + })""", + R"""( + root ::= "\"" ("a\-b") "\"" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); + + // a regexp that is invalid under any flavor is still an error + test({ + FAILURE, + "regexp with unbalanced parentheses", + R"""({ + "type": "string", + "pattern": "^(a$" + })""", + "" + }); + + // only the property with the bad pattern degrades + test({ + SUCCESS, + "unsupported regexp in a property", + R"""({ + "type": "object", + "properties": { + "a": { "type": "string", "pattern": "^(?=a)a$" } + }, + "required": ["a"], + "additionalProperties": false + })""", + R"""( + a ::= string + a-kv ::= "\"a\"" space ":" space a + char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) + root ::= "{" space a-kv space "}" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + string ::= "\"" char* "\"" + )""" + }); +} + +int main() { + test_all("JSON schema conversion", [](const TestCase & tc) { try { tc.verify(json_schema_to_grammar(common_json::parse(tc.schema), true)); tc.verify_status(SUCCESS); @@ -1525,127 +1547,58 @@ int main() { } }); - // C++ only tests (features not yet supported in JS/Python implementations) + // a document parsed up front gives the same grammar as the JSON, recursion included { - fprintf(stderr, "#\n# Testing C++ only features\n#\n"); - auto run = [](const TestCase & tc) { - fprintf(stderr, "- %s\n", tc.name.c_str()); - try { - tc.verify(json_schema_to_grammar(common_json::parse(tc.schema), true)); - tc.verify_status(SUCCESS); - } catch (const std::invalid_argument & ex) { - fprintf(stderr, "Error: %s\n", ex.what()); - tc.verify_status(FAILURE); + fprintf(stderr, "- parsed document\n"); + auto schema = common_json::parse(R"""({ + "$ref": "#/$defs/node", + "$defs": { + "node": { + "type": "object", + "properties": {"next": {"$ref": "#/$defs/node"}, "leaf": {}}, + "additionalProperties": false + } } - }; - - run({ - SUCCESS, - "regexp with non-capturing group", - R"""({ - "type": "string", - "pattern": "^(?:foo|bar)baz$" - })""", - R"""( - root ::= "\"" (("foo" | "bar") "baz") "\"" - space ::= | " " | "\n"{1,2} [ \t]{0,20} - )""", - }); - - run({ - SUCCESS, - "regexp with nested non-capturing groups", - R"""({ - "type": "string", - "pattern": "^(?:(?:ab)+c)?d$" - })""", - R"""( - root ::= "\"" ((("ab")+ "c")? "d") "\"" - space ::= | " " | "\n"{1,2} [ \t]{0,20} - )""", - }); - - run({ - SUCCESS, - "unanchored regexp", - R"""({ - "type": "string", - "pattern": "[0-9]+" - })""", - R"""( - char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) - root ::= string - space ::= | " " | "\n"{1,2} [ \t]{0,20} - string ::= "\"" char* "\"" - )""", - }); + })"""); + assert(json_schema_to_grammar(common_chat_schema_from_json(schema)) == json_schema_to_grammar(schema, true)); + } - // the rules of the partial conversion (here "root-0") must not leak into the grammar - run({ - SUCCESS, - "regexp with unsupported shorthand", - R"""({ - "type": "string", - "pattern": "^[0-9]{3}\\w$" - })""", - R"""( - char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) - root ::= string - space ::= | " " | "\n"{1,2} [ \t]{0,20} - string ::= "\"" char* "\"" - )""", - }); - - // a regexp that is invalid under any flavor is still an error - run({ - FAILURE, - "regexp with unbalanced parentheses", - R"""({ - "type": "string", - "pattern": "^(a$" - })""", - "" - }); - - // only the property with the bad pattern degrades - run({ + // a property node carries its $ref target, so its grammar names the ref rule + { + fprintf(stderr, "- sub-schema $ref\n"); + auto parameters = common_json::parse(R"""({ + "type": "object", + "properties": {"item": {"$ref": "#/$defs/item"}}, + "$defs": { + "item": { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + "additionalProperties": false + } + } + })"""); + TestCase tc { SUCCESS, - "unsupported regexp in a property", - R"""({ - "type": "object", - "properties": { - "a": { "type": "string", "pattern": "^[a-z\\-]+$" } - }, - "required": ["a"], - "additionalProperties": false - })""", + "sub-schema $ref", + "", R"""( - a ::= string - a-kv ::= "\"a\"" space ":" space a char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) - root ::= "{" space a-kv space "}" + ref-defs-item ::= "{" space ref-defs-item-a-kv space "}" + ref-defs-item-a-kv ::= "\"a\"" space ":" space string + root ::= ref-defs-item space ::= | " " | "\n"{1,2} [ \t]{0,20} string ::= "\"" char* "\"" )""", - }); - } - - if (getenv("LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR")) { - fprintf(stderr, "\033[33mWARNING: Skipping slow tests on emulator.\n\033[0m"); - } else { - if (getenv("LLAMA_PYTHON_AVAILABLE") || (std::system("python -c \"import sys; exit(1) if sys.version_info < (3, 8) else print('Python version is sufficient')\"") == 0)) { - test_all("Python", [](const TestCase & tc) { - write("test-json-schema-input.tmp", tc.schema); - tc.verify_status(std::system( - "python ./examples/json_schema_to_grammar.py test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); - tc.verify(read("test-grammar-output.tmp")); - }); - } else { - fprintf(stderr, "\033[33mWARNING: Python not found (min version required is 3.8), skipping Python JSON schema -> grammar tests.\n\033[0m"); - } + }; + auto doc = common_chat_schema_from_json(parameters); + tc.verify(build_grammar([&](const common_grammar_builder & builder) { + const auto & item = static_cast<const common_chat_schema_object &>(*doc.root).properties.at(0); + builder.add_schema("root", *item.schema); + })); } - test_all("Check Expectations Validity", [](const TestCase & tc) { + test_all("Check the expectations parse", [](const TestCase & tc) { if (tc.expected_status == SUCCESS) { tc.verify_expectation_parseable(); } diff --git a/tests/test-json-schema.cpp b/tests/test-json-schema.cpp new file mode 100644 index 000000000000..fb8cee18b9fb --- /dev/null +++ b/tests/test-json-schema.cpp @@ -0,0 +1,513 @@ +#include "json-schema.h" +#include "json.h" +#include "testing.h" + +#include <cstdlib> +#include <iostream> +#include <stdexcept> +#include <string> +#include <utility> + +static common_chat_schema_document parse(const std::string & schema) { + return common_chat_schema_from_json(common_json::parse(schema)); +} + +// the node as T, aborting the current test when it is some other kind +template <typename T> +static const T & as(testing & t, const common_chat_schema * node, const char * what) { + const T * typed = dynamic_cast<const T *>(node); + if (!t.assert_true(std::string(what) + " has the expected kind", typed != nullptr)) { + throw std::runtime_error(std::string(what) + " has the wrong kind"); + } + return *typed; +} + +template <typename T> +static const T & root(testing & t, const common_chat_schema_document & doc) { + return as<T>(t, doc.root.get(), "root"); +} + +static void assert_error(testing & t, const std::string & schema, const std::string & needle) { + try { + parse(schema); + t.assert_true(schema + " is rejected", false); + } catch (const std::runtime_error & e) { + std::string what = e.what(); + t.assert_true(schema + " -> " + what, what.find(needle) != std::string::npos); + } +} + +static void test_any(testing & t) { + t.test("empty schema", [](testing & t) { + auto doc = parse("{}"); + root<common_chat_schema_any>(t, doc); + t.assert_true("no refs", doc.refs.empty()); + }); + + t.test("keywords that do not imply a type", [](testing & t) { + auto doc = parse(R"({"description": "x", "format": "email", "additionalProperties": true})"); + root<common_chat_schema_any>(t, doc); + }); +} + +static void test_primitives(testing & t) { + t.test("null, boolean, number", [](testing & t) { + auto doc_null = parse(R"({"type": "null"})"); + root<common_chat_schema_null>(t, doc_null); + auto doc_bool = parse(R"({"type": "boolean"})"); + root<common_chat_schema_boolean>(t, doc_bool); + auto doc_num = parse(R"({"type": "number", "minimum": 1, "maximum": 2})"); + root<common_chat_schema_number>(t, doc_num); + }); +} + +static void test_integer(testing & t) { + t.test("unbounded", [](testing & t) { + auto doc = parse(R"({"type": "integer"})"); + const auto & i = root<common_chat_schema_integer>(t, doc); + t.assert_equal("minimum", INT64_MIN, i.minimum); + t.assert_equal("maximum", INT64_MAX, i.maximum); + }); + + t.test("inclusive bounds", [](testing & t) { + auto doc = parse(R"({"type": "integer", "minimum": -5, "maximum": 10})"); + const auto & i = root<common_chat_schema_integer>(t, doc); + t.assert_equal("minimum", -5, i.minimum); + t.assert_equal("maximum", 10, i.maximum); + }); + + t.test("exclusive bounds are folded", [](testing & t) { + auto doc = parse(R"({"type": "integer", "exclusiveMinimum": 0, "exclusiveMaximum": 10})"); + const auto & i = root<common_chat_schema_integer>(t, doc); + t.assert_equal("minimum", 1, i.minimum); + t.assert_equal("maximum", 9, i.maximum); + }); + + t.test("fractional bounds round inwards", [](testing & t) { + auto doc = parse(R"({"type": "integer", "minimum": 1.5, "exclusiveMaximum": 9.5})"); + const auto & i = root<common_chat_schema_integer>(t, doc); + t.assert_equal("minimum", 2, i.minimum); + t.assert_equal("maximum", 9, i.maximum); + }); +} + +static void test_string(testing & t) { + t.test("defaults", [](testing & t) { + auto doc = parse(R"({"type": "string"})"); + const auto & s = root<common_chat_schema_string>(t, doc); + t.assert_equal("pattern", "", s.pattern); + t.assert_equal("format", common_chat_schema::FORMAT_NONE, s.format); + t.assert_equal("min_length", 0, s.min_length); + t.assert_equal("max_length", -1, s.max_length); + }); + + t.test("all keywords are kept", [](testing & t) { + auto doc = parse(R"({"type": "string", "pattern": "^[a-z]+$", "format": "date", "minLength": 2, "maxLength": 8})"); + const auto & s = root<common_chat_schema_string>(t, doc); + t.assert_equal("pattern", "^[a-z]+$", s.pattern); + t.assert_equal("format", common_chat_schema::FORMAT_DATE, s.format); + t.assert_equal("min_length", 2, s.min_length); + t.assert_equal("max_length", 8, s.max_length); + }); + + t.test("formats", [](testing & t) { + auto expect = [&](const char * format, common_chat_schema::string_format expected) { + auto doc = parse(std::string(R"({"type": "string", "format": ")") + format + "\"}"); + t.assert_equal(format, expected, root<common_chat_schema_string>(t, doc).format); + }; + expect("time", common_chat_schema::FORMAT_TIME); + expect("date-time", common_chat_schema::FORMAT_DATE_TIME); + expect("uuid", common_chat_schema::FORMAT_UUID); + expect("uuid5", common_chat_schema::FORMAT_UUID); + expect("email", common_chat_schema::FORMAT_NONE); + }); + + t.test("pattern, length and known format imply a string", [](testing & t) { + auto doc_pattern = parse(R"({"pattern": "^a$"})"); + t.assert_equal("pattern", "^a$", root<common_chat_schema_string>(t, doc_pattern).pattern); + auto doc_length = parse(R"({"minLength": 1, "maxLength": 3})"); + t.assert_equal("min_length", 1, root<common_chat_schema_string>(t, doc_length).min_length); + t.assert_equal("max_length", 3, root<common_chat_schema_string>(t, doc_length).max_length); + auto doc_format = parse(R"({"format": "uuid"})"); + t.assert_equal("format", common_chat_schema::FORMAT_UUID, root<common_chat_schema_string>(t, doc_format).format); + }); +} + +static void test_array(testing & t) { + t.test("items with bounds", [](testing & t) { + auto doc = parse(R"({"type": "array", "items": {"type": "integer"}, "minItems": 1, "maxItems": 3})"); + const auto & a = root<common_chat_schema_array>(t, doc); + as<common_chat_schema_integer>(t, a.items.get(), "items"); + t.assert_equal("min_items", 1, a.min_items); + t.assert_equal("max_items", 3, a.max_items); + }); + + t.test("no items", [](testing & t) { + auto doc = parse(R"({"type": "array"})"); + const auto & a = root<common_chat_schema_array>(t, doc); + as<common_chat_schema_any>(t, a.items.get(), "items"); + t.assert_equal("min_items", 0, a.min_items); + t.assert_equal("max_items", -1, a.max_items); + }); + + t.test("items imply an array", [](testing & t) { + auto doc = parse(R"({"items": {"type": "string"}})"); + const auto & a = root<common_chat_schema_array>(t, doc); + as<common_chat_schema_string>(t, a.items.get(), "items"); + }); +} + +static void test_tuple(testing & t) { + t.test("prefixItems", [](testing & t) { + auto doc = parse(R"({"prefixItems": [{"type": "string"}, {"type": "number"}]})"); + const auto & tup = root<common_chat_schema_tuple>(t, doc); + t.assert_equal("size", (size_t) 2, tup.items.size()); + as<common_chat_schema_string>(t, tup.items[0].get(), "items[0]"); + as<common_chat_schema_number>(t, tup.items[1].get(), "items[1]"); + }); + + t.test("items as an array", [](testing & t) { + auto doc = parse(R"({"type": "array", "items": [{"type": "boolean"}]})"); + const auto & tup = root<common_chat_schema_tuple>(t, doc); + t.assert_equal("size", (size_t) 1, tup.items.size()); + as<common_chat_schema_boolean>(t, tup.items[0].get(), "items[0]"); + }); +} + +static void test_object(testing & t) { + t.test("type alone accepts any object", [](testing & t) { + auto doc = parse(R"({"type": "object"})"); + const auto & o = root<common_chat_schema_object>(t, doc); + t.assert_true("no properties", o.properties.empty()); + as<common_chat_schema_any>(t, o.additional_properties.get(), "additional_properties"); + }); + + t.test("properties", [](testing & t) { + auto doc = parse(R"({ + "type": "object", + "properties": { + "b": {"type": "string"}, + "a": {"type": "integer"}, + "c": {"type": "boolean"} + }, + "required": ["a", "c"] + })"); + const auto & o = root<common_chat_schema_object>(t, doc); + t.assert_equal("size", (size_t) 3, o.properties.size()); + t.assert_equal("order", "b", o.properties[0].name); + t.assert_equal("order", "a", o.properties[1].name); + t.assert_equal("order", "c", o.properties[2].name); + t.assert_true("b optional", !o.properties[0].required); + t.assert_true("a required", o.properties[1].required); + t.assert_true("c required", o.properties[2].required); + as<common_chat_schema_string>(t, o.properties[0].schema.get(), "b"); + as<common_chat_schema_integer>(t, o.properties[1].schema.get(), "a"); + as<common_chat_schema_boolean>(t, o.properties[2].schema.get(), "c"); + t.assert_true("closed", o.additional_properties == nullptr); + }); + + t.test("unknown required entries are ignored", [](testing & t) { + auto doc = parse(R"({"properties": {"a": {}}, "required": ["a", "zzz", 1]})"); + const auto & o = root<common_chat_schema_object>(t, doc); + t.assert_equal("size", (size_t) 1, o.properties.size()); + t.assert_true("a required", o.properties[0].required); + }); + + t.test("additionalProperties false implies an object", [](testing & t) { + auto doc = parse(R"({"additionalProperties": false})"); + const auto & o = root<common_chat_schema_object>(t, doc); + t.assert_true("no properties", o.properties.empty()); + t.assert_true("closed", o.additional_properties == nullptr); + }); + + t.test("additionalProperties schema", [](testing & t) { + auto doc = parse(R"({"properties": {"a": {}}, "additionalProperties": {"type": "integer", "minimum": 0}})"); + const auto & o = root<common_chat_schema_object>(t, doc); + t.assert_equal("size", (size_t) 1, o.properties.size()); + const auto & v = as<common_chat_schema_integer>(t, o.additional_properties.get(), "additional_properties"); + t.assert_equal("minimum", 0, v.minimum); + }); + + t.test("nested", [](testing & t) { + auto doc = parse(R"({"properties": {"inner": {"properties": {"leaf": {"type": "null"}}, "required": ["leaf"]}}})"); + const auto & o = root<common_chat_schema_object>(t, doc); + const auto & inner = as<common_chat_schema_object>(t, o.properties[0].schema.get(), "inner"); + t.assert_equal("leaf name", "leaf", inner.properties[0].name); + t.assert_true("leaf required", inner.properties[0].required); + as<common_chat_schema_null>(t, inner.properties[0].schema.get(), "leaf"); + }); +} + +static void test_const_enum(testing & t) { + t.test("const", [](testing & t) { + auto doc = parse(R"({"const": {"a": [1, null]}})"); + t.assert_equal("value", R"({"a":[1,null]})", root<common_chat_schema_const>(t, doc).value.dump()); + }); + + t.test("enum", [](testing & t) { + auto doc = parse(R"({"enum": ["a", 1, null, true]})"); + const auto & e = root<common_chat_schema_enum>(t, doc); + t.assert_equal("size", (size_t) 4, e.values.size()); + t.assert_equal("values[0]", "\"a\"", e.values[0].dump()); + t.assert_equal("values[1]", "1", e.values[1].dump()); + t.assert_equal("values[2]", "null", e.values[2].dump()); + t.assert_equal("values[3]", "true", e.values[3].dump()); + }); + + t.test("const wins over enum, enum wins over type", [](testing & t) { + auto doc_enum = parse(R"({"type": "integer", "enum": [1, 2]})"); + root<common_chat_schema_enum>(t, doc_enum); + auto doc_const = parse(R"({"type": "string", "const": "x", "enum": ["y"]})"); + t.assert_equal("value", "\"x\"", root<common_chat_schema_const>(t, doc_const).value.dump()); + }); +} + +static void test_any_of(testing & t) { + t.test("anyOf and oneOf", [](testing & t) { + auto doc_any = parse(R"({"anyOf": [{"type": "string"}, {"type": "number"}]})"); + const auto & u = root<common_chat_schema_any_of>(t, doc_any); + t.assert_equal("size", (size_t) 2, u.children.size()); + as<common_chat_schema_string>(t, u.children[0].get(), "children[0]"); + as<common_chat_schema_number>(t, u.children[1].get(), "children[1]"); + + auto doc_one = parse(R"({"oneOf": [{"type": "null"}]})"); + const auto & o = root<common_chat_schema_any_of>(t, doc_one); + t.assert_equal("size", (size_t) 1, o.children.size()); + as<common_chat_schema_null>(t, o.children[0].get(), "children[0]"); + }); + + t.test("oneOf wins over anyOf and type", [](testing & t) { + auto doc = parse(R"({"type": "string", "oneOf": [{"type": "null"}], "anyOf": [{"type": "number"}, {"type": "boolean"}]})"); + const auto & u = root<common_chat_schema_any_of>(t, doc); + t.assert_equal("size", (size_t) 1, u.children.size()); + as<common_chat_schema_null>(t, u.children[0].get(), "children[0]"); + }); + + t.test("type array expands with sibling keywords", [](testing & t) { + auto doc = parse(R"({"type": ["string", "null", "integer"], "minLength": 2, "minimum": 5})"); + const auto & u = root<common_chat_schema_any_of>(t, doc); + t.assert_equal("size", (size_t) 3, u.children.size()); + t.assert_equal("min_length", 2, as<common_chat_schema_string>(t, u.children[0].get(), "children[0]").min_length); + as<common_chat_schema_null>(t, u.children[1].get(), "children[1]"); + t.assert_equal("minimum", 5, as<common_chat_schema_integer>(t, u.children[2].get(), "children[2]").minimum); + }); +} + +static void test_all_of(testing & t) { + t.test("components", [](testing & t) { + auto doc = parse(R"({"allOf": [{"properties": {"a": {}}}, {"anyOf": [{"properties": {"b": {}}}, {"type": "null"}]}]})"); + const auto & all = root<common_chat_schema_all_of>(t, doc); + t.assert_equal("size", (size_t) 2, all.children.size()); + as<common_chat_schema_object>(t, all.children[0].get(), "children[0]"); + as<common_chat_schema_any_of>(t, all.children[1].get(), "children[1]"); + + auto doc_typed = parse(R"({"type": "object", "allOf": [{"properties": {"a": {}}}]})"); + root<common_chat_schema_all_of>(t, doc_typed); + }); + + t.test("properties win over allOf", [](testing & t) { + auto doc = parse(R"({"type": "object", "properties": {"a": {}}, "allOf": [{"properties": {"b": {}}}]})"); + t.assert_equal("size", (size_t) 1, root<common_chat_schema_object>(t, doc).properties.size()); + }); + + t.test("other types ignore allOf", [](testing & t) { + auto doc = parse(R"({"type": "integer", "allOf": [{"minimum": 1}]})"); + root<common_chat_schema_integer>(t, doc); + }); +} + +static void test_ref(testing & t) { + t.test("target is owned by the document", [](testing & t) { + auto doc = parse(R"({"$ref": "#/$defs/t", "type": "string", "$defs": {"t": {"type": "boolean"}}})"); + const auto & r = root<common_chat_schema_ref>(t, doc); + t.assert_equal("ref", "#/$defs/t", r.ref); + t.assert_equal("refs", (size_t) 1, doc.refs.size()); + t.assert_true("target", r.target != nullptr && r.target == doc.refs.at("#/$defs/t").get()); + as<common_chat_schema_boolean>(t, r.target, "target"); + }); + + t.test("definitions", [](testing & t) { + auto doc = parse(R"({"properties": {"a": {"$ref": "#/definitions/t"}}, "definitions": {"t": {"type": "number"}}})"); + const auto & o = root<common_chat_schema_object>(t, doc); + const auto & r = as<common_chat_schema_ref>(t, o.properties[0].schema.get(), "a"); + as<common_chat_schema_number>(t, r.target, "target"); + }); + + t.test("recursive", [](testing & t) { + auto doc = parse(R"({ + "$ref": "#/$defs/node", + "$defs": { + "node": { + "type": "object", + "properties": { + "value": {"type": "number"}, + "next": {"$ref": "#/$defs/node"} + }, + "required": ["value"] + } + } + })"); + const auto & r = root<common_chat_schema_ref>(t, doc); + const auto & node = as<common_chat_schema_object>(t, r.target, "node"); + t.assert_equal("properties", (size_t) 2, node.properties.size()); + const auto & next = as<common_chat_schema_ref>(t, node.properties[1].schema.get(), "next"); + t.assert_true("cycle", next.target == r.target); + t.assert_equal("refs", (size_t) 1, doc.refs.size()); + }); + + t.test("pointer through an array", [](testing & t) { + auto doc = parse(R"({"oneOf": [{"type": "null"}, {"$ref": "#/oneOf/0"}]})"); + const auto & u = root<common_chat_schema_any_of>(t, doc); + const auto & r = as<common_chat_schema_ref>(t, u.children[1].get(), "children[1]"); + as<common_chat_schema_null>(t, r.target, "target"); + }); + + t.test("targets survive moving the document", [](testing & t) { + auto parsed = parse(R"({"items": {"$ref": "#/$defs/t"}, "$defs": {"t": {"type": "null"}}})"); + common_chat_schema_document doc = std::move(parsed); + const auto & a = root<common_chat_schema_array>(t, doc); + const auto & r = as<common_chat_schema_ref>(t, a.items.get(), "items"); + t.assert_true("target", r.target == doc.refs.at("#/$defs/t").get()); + as<common_chat_schema_null>(t, r.target, "target"); + }); +} + +static void test_may_be_string(testing & t) { + auto check = [](testing & t, const std::string & schema, bool expected) { + t.assert_equal(schema, expected, parse(schema).root->may_be_string()); + }; + + t.test("leaves", [&](testing & t) { + check(t, R"({"type": "string"})", true); + check(t, R"({"type": "integer"})", false); + check(t, R"({"minLength": 1})", true); + check(t, R"({"pattern": "^[a-z]+$"})", true); + check(t, R"({"const": "hello"})", true); + check(t, R"({"const": 123})", false); + check(t, R"({"enum": [1, "a", null]})", true); + check(t, R"({"enum": [1, 2, 3]})", false); + }); + + t.test("composites", [&](testing & t) { + check(t, R"({"type": ["integer", "string"]})", true); + check(t, R"({"anyOf": [{"type": "integer"}, {"type": "boolean"}]})", false); + check(t, R"({"allOf": [{"type": "string"}, {"minLength": 1}]})", true); + check(t, R"({"allOf": [{"type": "string"}, {"type": "integer"}]})", false); + check(t, R"({"allOf": [{"minLength": 1}, {"maxLength": 2}]})", true); + }); + + t.test("ref", [&](testing & t) { + check(t, R"({"$ref": "#/$defs/n", "$defs": {"n": {"anyOf": [{"$ref": "#/$defs/n"}, {"type": "string"}]}}})", true); + check(t, R"({"$ref": "#/$defs/n", "$defs": {"n": {"$ref": "#/$defs/n"}}})", false); + check(t, R"({"anyOf": [{"$ref": "#/$defs/a"}, {"$ref": "#/$defs/b"}], "$defs": {"a": {"allOf": [{"$ref": "#/$defs/b"}, {"type": "integer"}]}, "b": {"type": "string"}}})", true); + }); +} + +// e.g. {number, integer}, in type order +static std::string dump(const common_chat_schema::type_set & types) { + static const common_chat_schema::value_type order[] = { common_chat_schema::TYPE_NULL, common_chat_schema::TYPE_BOOLEAN, common_chat_schema::TYPE_NUMBER, + common_chat_schema::TYPE_INTEGER, common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_ARRAY, + common_chat_schema::TYPE_OBJECT }; + std::string out; + for (auto type : order) { + if (types.has(type)) { + out += (out.empty() ? "" : ", ") + std::string(common_chat_schema::type_name(type)); + } + } + return "{" + out + "}"; +} + +static void test_value_types(testing & t) { + auto check = [](testing & t, const std::string & schema, const common_chat_schema::type_set & expected) { + t.assert_equal(schema, dump(expected), dump(parse(schema).root->value_types())); + }; + + t.test("leaves", [&](testing & t) { + check(t, R"({"type": "string"})", { common_chat_schema::TYPE_STRING }); + check(t, R"({"type": "number"})", { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER }); + check(t, R"({"description": "x"})", common_chat_schema::type_set::all()); + check(t, R"({"properties": {"a": {"type": "string"}}})", { common_chat_schema::TYPE_OBJECT }); + check(t, R"({"items": {"type": "string"}})", { common_chat_schema::TYPE_ARRAY }); + check(t, R"({"const": 1.5})", { common_chat_schema::TYPE_NUMBER }); + check(t, R"({"enum": [1, "a", null]})", { common_chat_schema::TYPE_INTEGER, common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_NULL }); + }); + + t.test("any_of is the union, all_of is the intersection", [&](testing & t) { + check(t, R"({"type": ["string", "null"]})", { common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_NULL }); + check(t, R"({"allOf": [{"type": ["string", "number"]}, {"type": ["number", "object"]}]})", { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER }); + check(t, R"({"allOf": [{"type": "string"}, {"type": "integer"}]})", {}); + }); + + t.test("ref", [&](testing & t) { + check(t, R"({"$ref": "#/$defs/n", "$defs": {"n": {"anyOf": [{"$ref": "#/$defs/n"}, {"type": "string"}]}}})", + { common_chat_schema::TYPE_STRING }); + }); +} + +static void test_errors(testing & t) { + t.test("not a schema", [](testing & t) { + assert_error(t, R"([])", "#: schema must be an object"); + }); + + t.test("type", [](testing & t) { + assert_error(t, R"({"type": 5})", "#: type must be a string or an array of strings"); + assert_error(t, R"({"type": []})", "#: type must not be empty"); + assert_error(t, R"({"type": ["string", "bad"]})", "#/type/1: unrecognized type bad"); + }); + + t.test("ref", [](testing & t) { + assert_error(t, R"({"$ref": 5})", "#: $ref must be a string"); + assert_error(t, R"({"$ref": "https://example.com/x.json"})", "#: unsupported $ref https://example.com/x.json"); + assert_error(t, R"({"$ref": ""})", "#: unsupported $ref ,"); + assert_error(t, R"({"$ref": "#"})", "#: unsupported $ref #,"); + assert_error(t, R"({"$defs": {}, "$ref": "#/$defs/missing"})", "#: cannot resolve $ref #/$defs/missing, missing not found"); + assert_error(t, R"({"oneOf": [{}], "$ref": "#/oneOf/1"})", "#: cannot resolve $ref #/oneOf/1, 1 is out of range"); + assert_error(t, R"({"$defs": {"a": {"$ref": "#/$defs/a/nope"}}, "$ref": "#/$defs/a"})", "#/$defs/a: cannot resolve $ref #/$defs/a/nope, nope not found"); + }); + + t.test("alternatives", [](testing & t) { + assert_error(t, R"({"oneOf": []})", "#/oneOf: must not be empty"); + assert_error(t, R"({"anyOf": {}})", "#/anyOf: must be an array of schemas"); + assert_error(t, R"({"anyOf": [{"type": "string"}, {"items": {"type": "x"}}]})", "#/anyOf/1/items: unrecognized type x"); + }); + + t.test("keywords", [](testing & t) { + assert_error(t, R"({"enum": []})", "#: enum must be a non-empty array"); + assert_error(t, R"({"type": "string", "pattern": 5})", "#: pattern must be a string"); + assert_error(t, R"({"type": "string", "minLength": -1})", "#: minLength must be a non-negative integer"); + assert_error(t, R"({"type": "integer", "minimum": "1"})", "#: minimum must be a number"); + assert_error(t, R"({"type": "array", "maxItems": 1.5})", "#: maxItems must be a non-negative integer"); + assert_error(t, R"({"properties": []})", "#: properties must be an object"); + assert_error(t, R"({"properties": {"a": {"type": "nope"}}})", "#/properties/a: unrecognized type nope"); + assert_error(t, R"({"additionalProperties": null})", "#: additionalProperties must be a boolean or a schema"); + }); +} + +int main(int argc, char * argv[]) { + testing t(std::cout); + if (argc >= 2) { + t.set_filter(argv[1]); + } + + const char * verbose = getenv("LLAMA_TEST_VERBOSE"); + if (verbose) { + t.verbose = std::string(verbose) == "1"; + } + + t.test("any", test_any); + t.test("primitives", test_primitives); + t.test("integer", test_integer); + t.test("string", test_string); + t.test("array", test_array); + t.test("tuple", test_tuple); + t.test("object", test_object); + t.test("const and enum", test_const_enum); + t.test("any_of", test_any_of); + t.test("all_of", test_all_of); + t.test("ref", test_ref); + t.test("may_be_string", test_may_be_string); + t.test("value_types", test_value_types); + t.test("errors", test_errors); + + return t.summary(); +} diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 0c33ac97d4fe..8218c96add0e 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -12,16 +12,25 @@ #include "../src/llama-model-saver.h" #include <cinttypes> +#include <cmath> #include <cstddef> #include <cstdio> #include <cstring> #include <cstdint> #include <random> +#include <regex> #include <stdexcept> #include <string> #include <utility> #include <vector> +static bool arch_matches(const std::string & filter, llm_arch arch) { + if (filter.empty()) { + return true; + } + return std::regex_search(llm_arch_name(arch), std::regex(filter)); +} + // normalized mean squared error = mse(a, b) / mse(a, 0) static double nmse(const std::vector<float> & a, const std::vector<float> & b) { GGML_ASSERT(a.size() == b.size()); @@ -39,24 +48,36 @@ static double nmse(const std::vector<float> & a, const std::vector<float> & b) { return mse_a_b / mse_a_0; } +struct tensor_data_params { + size_t seed; + float stdev; +}; + static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) { - size_t seed = *(const size_t *) userdata; + const tensor_data_params & params = *(const tensor_data_params *) userdata; + size_t seed = params.seed; std::hash<std::string> hasher; seed ^= hasher(tensor->name); std::mt19937 gen(seed); - std::normal_distribution<float> dis(0.0f, 1.0e-2f); + std::normal_distribution<float> dis(0.0f, params.stdev); + + // TODO: refactor per-tensor initialization logic in a cleaner way + // note: Mamba A must be negative (state decay) + const bool is_ssm_a = strstr(tensor->name, "ssm_a") != nullptr; const int64_t ne = ggml_nelements(tensor); if (tensor->type == GGML_TYPE_F32) { std::vector<float> tmp(ne); for (int64_t i = 0; i < ne; i++) { - tmp[i] = dis(gen); + float val = dis(gen); + tmp[i] = is_ssm_a ? -fabsf(val) : val; } ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor)); } else if (tensor->type == GGML_TYPE_F16) { std::vector<ggml_fp16_t> tmp(ne); for (int64_t i = 0; i < ne; i++) { - tmp[i] = ggml_fp32_to_fp16(dis(gen)); + float val = dis(gen); + tmp[i] = ggml_fp32_to_fp16(is_ssm_a ? -fabsf(val) : val); } ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor)); } else { @@ -65,7 +86,20 @@ static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) { } static void usage(char ** argv) { - printf("Usage: %s [-a/--arch arch] [-s/--seed seed] [-o/--out dir] [-v N] [-h/--help]\n", argv[0]); + LOG("Usage: %s [options]\n\n", argv[0]); + LOG("Options:\n"); + LOG(" -a, --arch <arch|regex> Run only matching LLM architectures (default: all supported)\n"); + LOG(" -s, --seed <seed> Set the random seed for tensor initialization and token generation\n"); + LOG(" -d, --stdev <stdev> Set the standard deviation of the tensor initialization distribution (default: 0.1f)\n"); + LOG(" -o, --out <dir> Save generated test models to <dir> instead of running backend tests\n"); + LOG(" -v <N> Set log verbosity level\n"); + LOG(" -b, --backend <backend> Run only on the given backend device\n"); + LOG(" -h, --help Show this help message\n\n"); + LOG("Examples:\n"); + LOG(" %s\n", argv[0]); + LOG(" %s -a qwen35moe\n", argv[0]); + LOG(" %s -a deepseek4 -o tests/test-models/\n", argv[0]); + LOG(" %s -a cohere2moe -v 5\n", argv[0]); } static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed){ @@ -128,7 +162,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { } else if (arch == LLM_ARCH_CHAMELEON) { n_vocab = 10240; } else if (arch == LLM_ARCH_QWEN3TTS) { - n_vocab = 4096; // must be >= the hard-coded codec head size (3072) + //n_vocab = 4096; // must be >= the hard-coded codec head size (3072) + n_vocab = 3072; // TODO: should be 4096, but user code cannot get `n_vocab_out` yet [TAG_LLAMA_N_VOCAB_OUT] + } else if (arch == LLM_ARCH_HRM_TEXT) { + n_layer = 8; // 1 layer per stack x 2 h-cycles x (3 l-cycles + 1) cache slots } uint32_t n_head_kv = n_head; @@ -237,8 +274,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f); // SWA pattern: every 5th layer is full attention (matches E2B layer_types) ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5)); - } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || - arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE) { + } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 || + arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE || + arch == LLM_ARCH_MAPLE) { std::vector<uint32_t> pattern; pattern.reserve(n_layer); for (uint32_t il = 0; il < n_layer; il++) { @@ -288,7 +326,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, arch == LLM_ARCH_QWEN4EXP ? n_embd_head : uint32_t(128)); - ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8)); + // note: using a realistic top-k here makes the results unstable and hard to match between CPU and GPU + // a large value makes things deterministic since all data is selected by the indexer + //ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8)); + ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(131072)); + ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4)); ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1)); ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4})); @@ -310,19 +352,43 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { } if (arch == LLM_ARCH_DEEPSEEK4) { - ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8)); - ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32)); - ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128})); - ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f); - ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4)); - ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2)); - ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f); + ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8)); + ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32)); + ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128})); + ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f); + ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4)); + ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2)); + ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f); ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0)); ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f); ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f); ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true); } - ms.add_kv(LLM_KV_TOKENIZER_MODEL, "no_vocab"); + + if (arch == LLM_ARCH_HRM_TEXT) { + // 8 cache slots alias 2 physical blocks: 1 low-stack layer + 1 high-stack layer + ms.add_kv(LLM_KV_HRM_LAYERS_PER_STACK, uint32_t(1)); + ms.add_kv(LLM_KV_HRM_H_CYCLES, uint32_t(2)); + ms.add_kv(LLM_KV_HRM_L_CYCLES, uint32_t(3)); + } + + if (arch == LLM_ARCH_MAPLE) { + ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f); + } + + // dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids + { + std::vector<std::string> tokenizer_list(n_vocab); + std::vector<float> tokenizer_scores(n_vocab, 0.0f); + + ms.add_kv(LLM_KV_TOKENIZER_MODEL, "test"); + for (uint32_t i = 0; i < n_vocab; i++) { + tokenizer_list[i] = "tok_" + std::to_string(i); + } + ms.add_kv(LLM_KV_TOKENIZER_LIST, tokenizer_list); + ms.add_kv(LLM_KV_TOKENIZER_SCORES, tokenizer_scores); + } + // ms.add_kv(LLM_KV_DENSE_2_FEAT_OUT, n_embd); // ms.add_kv(LLM_KV_DENSE_3_FEAT_IN, n_embd); @@ -332,7 +398,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_EXPERT_LATENT_LENGTH, n_ff); ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2)); ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(2)); - ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(1)); + ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(2)); ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1)); ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(4) : uint32_t(2)); // sqrtsoftplus : sigmoid ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f); @@ -353,19 +419,19 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head); ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2)); ms.add_kv(LLM_KV_KDA_HEAD_DIM, uint32_t(128)); - ms.add_kv(LLM_KV_KDA_SAFE_GATE, true); - ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f); + ms.add_kv(LLM_KV_KDA_SAFE_GATE, true); + ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f); if (arch == LLM_ARCH_BAILINGMOE3) { ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>({0.0f, 4.0f})); ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>({0.0f, 5.0f})); } - ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head); - ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3)); - ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f); - ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12)); - ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f); + ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head); + ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3)); + ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f); + ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12)); + ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f); ms.add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, 25.0f); - ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f); + ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f); for (uint32_t il = 0; il < n_layer; il++) { ggml_tensor t; @@ -388,7 +454,8 @@ static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/) } static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx( - struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const std::vector<ggml_backend_dev_t> & devs, + struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev, + const std::vector<ggml_backend_dev_t> & devs, const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false) { GGML_ASSERT((gguf_ctx == nullptr) != (file == nullptr)); llama_model_params model_params = llama_model_default_params(); @@ -406,9 +473,9 @@ static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx( ctx_params.n_ubatch = 64; } - size_t tmp = seed; + tensor_data_params tensor_params = { seed, stdev }; llama_model_ptr model(gguf_ctx != nullptr ? - llama_model_init_from_user(gguf_ctx, set_tensor_data, &tmp, model_params) : + llama_model_init_from_user(gguf_ctx, set_tensor_data, &tensor_params, model_params) : llama_model_load_from_file_ptr(file, model_params)); if (!model) { throw std::runtime_error("failed to create llama model"); @@ -505,6 +572,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_MISTRAL4: case LLM_ARCH_MELLUM: case LLM_ARCH_LAGUNA: + case LLM_ARCH_MAPLE: return true; default: return false; @@ -577,7 +645,8 @@ static bool arch_supported(const llm_arch arch) { // FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI. #ifdef GGML_USE_WEBGPU - if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_QWEN4EXP) { + if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_QWEN4EXP || + arch == LLM_ARCH_HY_V4) { return false; } #endif // GGML_USE_WEBGPU @@ -593,7 +662,7 @@ static bool arch_supported(const llm_arch arch) { return true; } -static int save_models(const llm_arch target_arch, const size_t seed, const int verbosity, const std::string & dir) { +static int save_models(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const std::string & dir) { struct user_data_t { struct { ggml_log_callback callback; @@ -620,7 +689,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const int if (arch == LLM_ARCH_UNKNOWN) { continue; } - if (target_arch != LLM_ARCH_UNKNOWN && arch != target_arch) { + if (!arch_matches(arch_filter, arch)) { continue; } if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) { @@ -641,7 +710,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const int continue; } gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe); - auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {}); + auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {}); const std::string path = dir + "/" + llm_arch_name(arch) + (moe ? "-moe.gguf" : "-dense.gguf"); LOG_INF("%s: Saving %s model (%s) to %s...\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense", path.c_str()); llama_model_save_to_file(model_and_ctx.first.get(), path.c_str()); @@ -651,7 +720,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const int return 0; } -static int test_backends(const llm_arch target_arch, const size_t seed, const int verbosity) { +static int test_backends(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const char * target_backend) { struct user_data_t { struct { ggml_log_callback callback; @@ -693,6 +762,9 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in const size_t device_count = ggml_backend_dev_count(); for (size_t i = 0; i < device_count; i++) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); + if (target_backend != nullptr && strcmp(target_backend, ggml_backend_dev_name(dev)) != 0) { + continue; + } dev_configs.emplace_back(std::vector<ggml_backend_dev_t>{dev}, ggml_backend_dev_description(dev), LLAMA_SPLIT_MODE_LAYER); max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length()); @@ -703,7 +775,9 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in } } - dev_configs.emplace_back(devices_meta, "Meta", LLAMA_SPLIT_MODE_TENSOR); + if (target_backend == nullptr) { + dev_configs.emplace_back(devices_meta, "Meta", LLAMA_SPLIT_MODE_TENSOR); + } } size_t max_arch_name_length = 0; @@ -716,22 +790,24 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in const std::string template_row_res = "%15s %10s|%20s|\n"; bool all_ok = true; + size_t n_tests = 0; + size_t n_failed = 0; common_log_flush(common_log_main()); - printf(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip"); - printf("|"); + LOG(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip"); + LOG("|"); for (size_t i = 0; i < max_arch_name_length; i++) { - printf("-"); + LOG("-"); } - printf("|"); + LOG("|"); for (size_t i = 0; i < max_device_label_length; i++) { - printf("-"); + LOG("-"); } - printf("|------|---------------|---------|\n"); + LOG("|------|---------------|---------|\n"); for (const llm_arch & arch : llm_arch_all()) { if (arch == LLM_ARCH_UNKNOWN) { continue; } - if (target_arch != LLM_ARCH_UNKNOWN && arch != target_arch) { + if (!arch_matches(arch_filter, arch)) { continue; } if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) { @@ -758,8 +834,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in std::vector<float> logits_cpu; for (device_config & dc : dev_configs) { // print test config first; should anything fail during model loading or inference, at least we know which test case caused it - printf(template_row_cfg.c_str(), - llm_arch_name(arch), dc.label.c_str(), config_name.c_str()); + LOG(template_row_cfg.c_str(), llm_arch_name(arch), dc.label.c_str(), config_name.c_str()); fflush(stdout); std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_dev; @@ -769,19 +844,22 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in char nmse_str[12] = {0}; bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty()); + bool test_executed = false; + bool test_ok = true; if (!skip) { if (logits_cpu.empty()) { - model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {}, LLAMA_SPLIT_MODE_LAYER, encode); + model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {}, LLAMA_SPLIT_MODE_LAYER, encode); logits_cpu = get_logits(model_and_ctx_cpu.first.get(), model_and_ctx_cpu.second.get(), tokens, encode); } if (dc.split_mode != LLAMA_SPLIT_MODE_TENSOR || llm_arch_supports_sm_tensor(arch)) { - model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, dc.devs, dc.split_mode, encode); + test_executed = true; + model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, dc.devs, dc.split_mode, encode); logits_dev = get_logits(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens, encode); const double nmse_val = nmse(logits_cpu, logits_dev); snprintf(nmse_str, sizeof(nmse_str), "(%.2e)", nmse_val); status_nmse = "\033[1;32mOK\033[0m"; if (nmse_val > 1e-4) { - all_ok = false; + test_ok = false; status_nmse = "\033[1;31mFAIL\033[0m"; } } @@ -790,6 +868,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in // FIXME: when adding a tensor to a gguf_context a copy is made, this changes the pointer which the meta backend // in turn uses to map the tensors to their simple equivalents - this is fundamentally incompatible if (file != nullptr && llama_model_saver_supports_arch(arch) && dc.split_mode != LLAMA_SPLIT_MODE_TENSOR) { + test_executed = true; GGML_ASSERT(model_and_ctx_dev.first && model_and_ctx_dev.second); llama_model_saver ms = llama_model_saver(model_and_ctx_dev.first.get()); ms.add_kv_from_model(); @@ -797,14 +876,14 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in ms.save(file); rewind(file); - auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, dc.devs, dc.split_mode, encode); + auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, stdev, dc.devs, dc.split_mode, encode); const std::vector<float> logits_roundtrip = get_logits( model_and_ctx_roundtrip.first.get(), model_and_ctx_roundtrip.second.get(), tokens, encode); status_roundtrip = "\033[1;32mOK\033[0m"; GGML_ASSERT(logits_roundtrip.size() == logits_dev.size()); for (size_t i = 0; i < logits_roundtrip.size(); i++) { if (logits_roundtrip[i] != logits_dev[i]) { - all_ok = false; + test_ok = false; status_roundtrip = "\033[1;31mFAIL\033[0m"; break; } @@ -812,12 +891,28 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in } } + if (test_executed) { + n_tests++; + if (!test_ok) { + n_failed++; + all_ok = false; + } + } + // log the results for this test case - printf(template_row_res.c_str(), - status_nmse.c_str(), nmse_str, status_roundtrip.c_str()); + LOG(template_row_res.c_str(), status_nmse.c_str(), nmse_str, status_roundtrip.c_str()); } } } + + if (n_tests == 0) { + LOG("Summary: no tests executed\n"); + } else if (n_failed == 0) { + LOG("Summary: all %zu test(s) passed\n", n_tests); + } else { + LOG("Summary: %zu test(s) executed, %zu failed\n", n_tests, n_failed); + } + llama_log_set(ud.log_old.callback, ud.log_old.user_data); return all_ok ? 0 : 1; } @@ -829,9 +924,11 @@ int main(int argc, char ** argv) { std::random_device rd; - llm_arch arch = LLM_ARCH_UNKNOWN; + std::string arch_filter; size_t seed = rd(); + float stdev = 0.1f; std::string out; + const char * target_backend = nullptr; int verbosity = LOG_LEVEL_ERROR; @@ -839,52 +936,83 @@ int main(int argc, char ** argv) { if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { usage(argv); return 0; - } - if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) { + } else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) { if (i + 1 < argc) { const std::string arch_name = argv[++i]; - arch = llm_arch_from_string(arch_name); - if (arch == LLM_ARCH_UNKNOWN) { - LOG_ERR("%s: unkown LLM architecture: %s\n", __func__, arch_name.c_str()); - return 1; + if (llm_arch_from_string(arch_name) != LLM_ARCH_UNKNOWN) { + // exact architecture name + arch_filter = "^" + arch_name + "$"; + } else { + try { + std::regex re(arch_name); + arch_filter = arch_name; + } catch (const std::regex_error & err) { + LOG_ERR("%s: invalid architecture regex: %s (%s)\n", __func__, arch_name.c_str(), err.what()); + return 1; + } } } else { usage(argv); return 1; } - } - if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) { + } else if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) { if (i + 1 < argc) { seed = std::stoull(argv[++i]); } else { usage(argv); return 1; } - } - if (strcmp(argv[i], "-v") == 0) { + } else if (strcmp(argv[i], "-d") == 0 || strcmp(argv[i], "--stdev") == 0) { + if (i + 1 < argc) { + stdev = std::stof(argv[++i]); + } else { + usage(argv); + return 1; + } + } else if (strcmp(argv[i], "-v") == 0) { if (i + 1 < argc) { verbosity = std::stoull(argv[++i]); } else { usage(argv); return 1; } - } - if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) { + } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) { if (i + 1 < argc) { out = argv[++i]; } else { usage(argv); return 1; } + } else if (strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "--backend") == 0) { + if (i + 1 < argc) { + const char * backend_name = argv[++i]; + ggml_backend_dev_t dev = ggml_backend_dev_by_name(backend_name); + if (dev == nullptr) { + LOG_ERR("%s: unknown backend device: %s\n", __func__, backend_name); + return 1; + } + target_backend = ggml_backend_dev_name(dev); + } else { + usage(argv); + return 1; + } + } else { + LOG_ERR("%s: unknown argument: %s\n", __func__, argv[i]); + usage(argv); + return 1; } } - printf("%s: using seed %zu\n", __func__, seed); + if (stdev <= 0.0f) { + LOG_ERR("%s: stdev must be > 0\n", __func__); + return 1; + } + LOG_INF("%s: using seed %zu, stdev %f\n", __func__, seed, stdev); try { if (!out.empty()) { - return save_models(arch, seed, verbosity, out); + return save_models(arch_filter, seed, stdev, verbosity, out); } - return test_backends(arch, seed, verbosity); + return test_backends(arch_filter, seed, stdev, verbosity, target_backend); } catch (const std::exception & err) { fprintf(stderr, "encountered runtime error: %s\n", err.what()); return -1; diff --git a/tests/test-quant-type-selection.cpp b/tests/test-quant-type-selection.cpp index 9a5f5e53e119..1696ec164c07 100644 --- a/tests/test-quant-type-selection.cpp +++ b/tests/test-quant-type-selection.cpp @@ -221,7 +221,7 @@ static const remote_model_spec model_specs[] = { { "ggml-org/Step-3.5-Flash-GGUF", "Q4_K" }, { "ggml-org/Qwen3-Coder-Next-GGUF", "Q8_0" }, { "ggml-org/Qwen3-14B-GGUF", "Q8_0" }, - { "ggml-org/NVIDIA-Nemotron-Nano-3-30B-A3B-GGUF", "Q8_0" }, + { "ggml-org/NVIDIA-Nemotron-3-Nano-30B-A3B-GGUF", "Q8_0" }, { "ggml-org/gpt-oss-120b-GGUF", "mxfp4" }, { "ggml-org/gemma-3-4b-it-GGUF", "Q8_0" }, { "bartowski/Meta-Llama-3.1-70B-Instruct-GGUF", "Q4_K_M" }, diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 9510ac14ce00..570fca89a876 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -5,6 +5,8 @@ #undef NDEBUG #include <assert.h> +#include <algorithm> +#include <cmath> #include <math.h> #include <stdio.h> #include <string> @@ -32,9 +34,9 @@ static const char* RESULT_STR[] = {"ok", "FAILED"}; // Generate synthetic data -static void generate_data(float offset, size_t n, float * dst) { +static void generate_data(float offset, size_t n, float * dst, float amplitude = 2.0f) { for (size_t i = 0; i < n; i++) { - dst[i] = 0.1 + 2*cosf(i + offset); + dst[i] = 0.1 + amplitude*cosf(i + offset); } } @@ -83,23 +85,50 @@ static float dot_product(const float * a1, const float * a2, size_t test_size) { } // Total dot product error -static float dot_product_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data1, const float * test_data2) { - GGML_UNUSED(qfns); - - std::vector<uint8_t> tmp_q1(2*test_size); - std::vector<uint8_t> tmp_q2(2*test_size); - +static float dot_product_error(const ggml_type_traits_cpu * qfns_cpu, ggml_type src0_type, size_t test_size, + const float * test_data1, const float * test_data2, + const float * test_data3, const float * test_data4, + const int nrc) { const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type); + const size_t pad = 64; + const size_t bx = ggml_row_size(src0_type, test_size) + pad; + const size_t by = ggml_row_size(qfns_cpu->vec_dot_type, test_size) + pad; + + std::vector<uint8_t> tmp_q1(bx * nrc); + std::vector<uint8_t> tmp_q2(by * nrc); qfns_cpu->from_float(test_data1, tmp_q1.data(), test_size); vdot->from_float(test_data2, tmp_q2.data(), test_size); - float result = INFINITY; - qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1); + if (nrc == 1) { + float result = INFINITY; + qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1); + + const float dot_ref = dot_product(test_data1, test_data2, test_size); + return fabsf(result - dot_ref) / test_size; + } + + // nrc == 2: kernel computes a 2x2 dot product matrix + // Output layout: s[0]=dot(vx0,vy0), s[1]=dot(vx1,vy0), s[bs]=dot(vx0,vy1), s[bs+1]=dot(vx1,vy1) + // row and output strides are padded, same as in the mul_mat path + qfns_cpu->from_float(test_data3, tmp_q1.data() + bx, test_size); + vdot->from_float(test_data4, tmp_q2.data() + by, test_size); + + const size_t bs = 16; + std::vector<float> result(bs + 2, INFINITY); + qfns_cpu->vec_dot(test_size, result.data(), bs, tmp_q1.data(), bx, tmp_q2.data(), by, 2); + + const float ref00 = dot_product(test_data1, test_data2, test_size); + const float ref10 = dot_product(test_data3, test_data2, test_size); + const float ref01 = dot_product(test_data1, test_data4, test_size); + const float ref11 = dot_product(test_data3, test_data4, test_size); - const float dot_ref = dot_product(test_data1, test_data2, test_size); + const auto err = [test_size](float val, float ref) { + const float e = fabsf(val - ref) / test_size; + return std::isfinite(e) ? e : INFINITY; + }; - return fabsf(result - dot_ref) / test_size; + return std::max({err(result[0], ref00), err(result[1], ref10), err(result[bs], ref01), err(result[bs + 1], ref11)}); } static int test_vec_dot_f32(bool verbose) { @@ -133,9 +162,13 @@ static int test_vec_dot_q(bool verbose) { std::vector<float> test_data(test_size); std::vector<float> test_data2(test_size); + std::vector<float> test_data3(test_size); + std::vector<float> test_data4(test_size); generate_data(0.0, test_data.size(), test_data.data()); generate_data(1.0, test_data2.size(), test_data2.data()); + generate_data(3.0, test_data3.size(), test_data3.data(), 1.0f); + generate_data(4.0, test_data4.size(), test_data4.data(), 1.5f); for (int i = 0; i < GGML_TYPE_COUNT; i++) { ggml_type type = (ggml_type) i; @@ -178,7 +211,7 @@ static int test_vec_dot_q(bool verbose) { printf("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error); } - const float vec_dot_error = dot_product_error(qfns, qfns_cpu, test_size, test_data.data(), test_data2.data()); + const float vec_dot_error = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), nullptr, nullptr, 1); const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S ? MAX_DOT_PRODUCT_ERROR_LOWBIT @@ -194,6 +227,16 @@ static int test_vec_dot_q(bool verbose) { if (failed || verbose) { printf("%5s dot product error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error); } + + // Test nrc=2 path for types that support it + if (qfns_cpu->nrows == 2) { + const float vec_dot_error_nrc2 = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), test_data3.data(), test_data4.data(), 2); + failed = !(vec_dot_error_nrc2 < max_allowed_error); + num_failed += failed; + if (failed || verbose) { + printf("%5s dot product error (nrc=2): %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error_nrc2); + } + } } } diff --git a/tests/test-recurrent-state-rollback.cpp b/tests/test-recurrent-state-rollback.cpp index c6f599e584ca..fdac344d8990 100644 --- a/tests/test-recurrent-state-rollback.cpp +++ b/tests/test-recurrent-state-rollback.cpp @@ -1,22 +1,19 @@ #include "arg.h" #include "common.h" +#include "ggml-backend.h" #include "llama.h" +#include "../src/llama-io.h" +#include "../src/llama-memory.h" + #include <algorithm> #include <clocale> #include <cmath> #include <cstdio> +#include <limits> +#include <set> #include <vector> -static llama_context * make_ctx(const common_params & params, llama_model * model) { - auto cparams = common_context_params_to_llama(params); - cparams.n_seq_max = 1; - cparams.n_rs_seq = 8; - cparams.n_batch = std::max(cparams.n_batch, (uint32_t) (cparams.n_rs_seq + 1)); - cparams.n_ubatch = std::max(cparams.n_ubatch, (uint32_t) (cparams.n_rs_seq + 1)); - return llama_init_from_model(model, cparams); -} - static bool decode_tokens(llama_context * ctx, const std::vector<llama_token> & tokens, uint32_t count) { llama_batch batch = llama_batch_init(count, 0, 1); for (uint32_t pos = 0; pos < count; ++pos) { @@ -35,12 +32,84 @@ static bool decode_one(llama_context * ctx, llama_token tok, llama_pos pos) { return ok; } +struct cache_buffer_collector : llama_io_write_i { + std::set<ggml_backend_buffer_t> buffers; + size_t size = 0; + + void write(const void *, size_t n) override { + size += n; + } + + void write_tensor(ggml_tensor * tensor, size_t, size_t n) override { + buffers.insert(tensor->buffer); + size += n; + } + + size_t n_bytes() override { + return size; + } +}; + +static llama_context * init_ctx(llama_model * model, llama_context_params cparams, uint8_t fill) { + llama_context * ctx = llama_init_from_model(model, cparams); + if (ctx == nullptr || fill == 0) { + return ctx; + } + + // Use a full ubatch so buffer discovery preserves prefill allocation sizes. + const uint32_t n_tokens = llama_n_ubatch(ctx); + if (!decode_tokens(ctx, std::vector<llama_token>(n_tokens, 0), n_tokens)) { + llama_free(ctx); + return nullptr; + } + llama_synchronize(ctx); + cache_buffer_collector collector; + llama_get_memory(ctx)->state_write(collector); + llama_memory_clear(llama_get_memory(ctx), true); + if (collector.buffers.empty()) { + fprintf(stderr, "%s : no cache buffers found\n", __func__); + llama_free(ctx); + return nullptr; + } + for (auto * buffer : collector.buffers) { + ggml_backend_buffer_clear(buffer, fill); + } + return ctx; +} + +static llama_context * make_ctx(const common_params & params, llama_model * model, uint8_t fill) { + auto cparams = common_context_params_to_llama(params); + cparams.n_seq_max = 1; + cparams.n_rs_seq = 8; + cparams.n_batch = std::max(cparams.n_batch, (uint32_t) (cparams.n_rs_seq + 1)); + cparams.n_ubatch = std::max(cparams.n_ubatch, (uint32_t) (cparams.n_rs_seq + 1)); + return init_ctx(model, cparams, fill); +} + +static float logit_diff(float a, float b) { + return std::isfinite(a) && std::isfinite(b) ? std::fabs(a - b) : std::numeric_limits<float>::infinity(); +} + +static double nmse(const float * a, const float * b, int n) { + double mse_ab = 0.0; + double mse_a0 = 0.0; + for (int i = 0; i < n; i++) { + if (!std::isfinite(a[i]) || !std::isfinite(b[i])) { + return std::numeric_limits<double>::infinity(); + } + const double diff = (double) a[i] - b[i]; + mse_ab += diff*diff; + mse_a0 += (double) a[i]*a[i]; + } + return mse_a0 == 0.0 ? (mse_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : mse_ab/mse_a0; +} + // Roll back multiple sequences, then replay them in a single batch whose // per-seq token count exceeds n_ubatch: each seq's replay spans several // ubatches while its rollback restore is still pending. Compared against a // reference context that never advanced past the rollback point and decodes // the identical replay batch. -static bool test_multi_seq_split_replay(const common_params & params, llama_model * model, const int n_vocab) { +static bool test_multi_seq_split_replay(const common_params & params, llama_model * model, const int n_vocab, uint8_t fill) { constexpr uint32_t n_seqs = 2; constexpr uint32_t n_ubatch = 16; constexpr uint32_t n_prompt = 19; @@ -56,7 +125,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode cparams.n_batch = 256; cparams.n_ubatch = n_ubatch; cparams.kv_unified = false; - return llama_init_from_model(model, cparams); + return init_ctx(model, cparams, fill); }; llama_context * ctx_roll = make_ctx_multi(); @@ -127,13 +196,15 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode return false; } - // identical ubatch shapes from bit-exact states: a correct implementation - // matches bitwise, so eps only allows backend scheduling noise - constexpr float eps = 1e-7f; + // identical ubatch shapes should produce identical states, but the larger + // stdev makes the model sensitive to backend scheduling/rounding noise + constexpr float nmse_eps = 1e-5f; float diff_max = 0.0f; uint32_t seq_first = 0; int32_t pos_first = -1; + double nmse_ab = 0.0; + double nmse_a0 = 0.0; for (uint32_t i = 0; i < n_seqs*n_replay; ++i) { const float * l_roll = llama_get_logits_ith(ctx_roll, i); const float * l_ref = llama_get_logits_ith(ctx_ref, i); @@ -143,23 +214,34 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode return false; } for (int t = 0; t < n_vocab; ++t) { - const float diff = std::fabs(l_roll[t] - l_ref[t]); - if (diff > eps && pos_first < 0) { + const float r = l_roll[t]; + const float f = l_ref[t]; + const float diff = logit_diff(r, f); + if (diff > 0.0f && pos_first < 0) { seq_first = i/n_replay; pos_first = p0 + (int32_t) (i%n_replay); } diff_max = std::max(diff_max, diff); + if (std::isfinite(r) && std::isfinite(f)) { + const double d = (double) r - f; + nmse_ab += d*d; + nmse_a0 += (double) r*r; + } else { + nmse_ab = std::numeric_limits<double>::infinity(); + nmse_a0 = 1.0; + } } } + const double nmse_val = nmse_a0 == 0.0 ? (nmse_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : nmse_ab/nmse_a0; - if (diff_max > eps) { - fprintf(stderr, "%s : multi-seq split replay logits mismatch (max diff %g, first at seq %u pos %d)\n", - __func__, (double) diff_max, seq_first, pos_first); + if (nmse_val > nmse_eps) { + fprintf(stderr, "%s : multi-seq split replay logits mismatch (max diff %g, nmse %g, first at seq %u pos %d)\n", + __func__, (double) diff_max, nmse_val, seq_first, pos_first); cleanup(); return false; } - fprintf(stderr, "%s : multi-seq split replay matched (max diff %g)\n", __func__, (double) diff_max); + fprintf(stderr, "%s : multi-seq split replay matched (max diff %g, nmse %g)\n", __func__, (double) diff_max, nmse_val); // seq-1-only decodes must be independent of seq 0's content: diverge seq 0 // in ctx_ref only, then compare identical seq-1-only continuations bitwise @@ -176,6 +258,8 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode } float diff_tail = 0.0f; + double nmse_tail_ab = 0.0; + double nmse_tail_a0 = 0.0; for (uint32_t i = 0; i < n_tail && ok; ++i) { const llama_pos pos = p0 + (llama_pos) (n_replay + i); llama_batch batch_one = llama_batch_init(1, 0, 1); @@ -191,54 +275,40 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode const float * l_ref = llama_get_logits_ith(ctx_ref, 0); ok = l_roll != nullptr && l_ref != nullptr; for (int t = 0; ok && t < n_vocab; ++t) { - diff_tail = std::max(diff_tail, std::fabs(l_roll[t] - l_ref[t])); + const float r = l_roll[t]; + const float f = l_ref[t]; + diff_tail = std::max(diff_tail, logit_diff(r, f)); + if (std::isfinite(r) && std::isfinite(f)) { + const double d = (double) r - f; + nmse_tail_ab += d*d; + nmse_tail_a0 += (double) r*r; + } else { + nmse_tail_ab = std::numeric_limits<double>::infinity(); + nmse_tail_a0 = 1.0; + } } } + const double nmse_tail = nmse_tail_a0 == 0.0 ? (nmse_tail_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : nmse_tail_ab/nmse_tail_a0; - if (!ok || diff_tail > eps) { - fprintf(stderr, "%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g)\n", - __func__, ok ? 1 : 0, (double) diff_tail); + if (!ok || nmse_tail > nmse_eps) { + fprintf(stderr, "%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g, nmse %g)\n", + __func__, ok ? 1 : 0, (double) diff_tail, nmse_tail); cleanup(); return false; } - fprintf(stderr, "%s : seq-1-only decode independent of seq 0 (max diff %g)\n", __func__, (double) diff_tail); + fprintf(stderr, "%s : seq-1-only decode independent of seq 0 (max diff %g, nmse %g)\n", __func__, (double) diff_tail, nmse_tail); cleanup(); return true; } -int main(int argc, char ** argv) { - std::setlocale(LC_NUMERIC, "C"); - - common_params params; - params.sampling.seed = 1234; - params.n_predict = 1; - - common_init(); - - if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { - return 1; - } - - ggml_backend_load_all(); - - common_init_result_ptr llama_init = common_init_from_params(params); - llama_model * model = llama_init->model(); - if (model == nullptr) { - fprintf(stderr, "%s : failed to init model\n", __func__); - return 1; - } - - if (!llama_model_is_recurrent(model) && !llama_model_is_hybrid(model)) { - fprintf(stderr, "%s : skipping for non-recurrent model\n", __func__); - return 0; - } - +static int test_rollback(const common_params & params, llama_model * model, uint8_t fill) { const llama_vocab * vocab = llama_model_get_vocab(model); const int n_vocab = llama_vocab_n_tokens(vocab); - llama_context * ctx_src = make_ctx(params, model); - llama_context * ctx_dst = make_ctx(params, model); + // TODO: use smart pointers + llama_context * ctx_src = make_ctx(params, model, fill); + llama_context * ctx_dst = make_ctx(params, model, fill); if (ctx_src == nullptr || ctx_dst == nullptr) { fprintf(stderr, "%s : failed to init contexts\n", __func__); return 1; @@ -291,7 +361,7 @@ int main(int argc, char ** argv) { ckpt.update_tgt(ctx_src, 0, 0); ckpt.load_tgt(ctx_dst, 0, 0); - constexpr float eps = 1e-5f; + constexpr float nmse_eps = 0.0; std::vector<std::vector<float>> logits_src_replay(n_rollback); const auto replay_and_compare = [&](const char * mode) { for (uint32_t i = 0; i < n_rollback; ++i) { @@ -310,13 +380,18 @@ int main(int argc, char ** argv) { } logits_src_replay[i].assign(logits_src, logits_src + n_vocab); + const double nmse_val = nmse(logits_src, logits_dst, n_vocab); + int token_first = -1; for (int token = 0; token < n_vocab; ++token) { - if (std::fabs(logits_src[token] - logits_dst[token]) > eps) { - fprintf(stderr, "%s : %s logits mismatch at position %d, token %d (%g != %g)\n", - __func__, mode, pos, token, (double) logits_src[token], (double) logits_dst[token]); - return false; + if (logit_diff(logits_src[token], logits_dst[token]) > 0.0f && token_first < 0) { + token_first = token; } } + if (nmse_val > nmse_eps) { + fprintf(stderr, "%s : %s logits mismatch at position %d, first token %d, nmse %g\n", + __func__, mode, pos, token_first, nmse_val); + return false; + } } return true; }; @@ -324,25 +399,27 @@ int main(int argc, char ** argv) { return 1; } - if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) || - !llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) { - fprintf(stderr, "%s : partial rollback failed\n", __func__); - return 1; - } + // TODO: this test is invalid because RS rollback is only correct once after a ubatch with more than n_rs_seq tokens + // this is not the case here. add asserts and guardrails to prevent such attempts + //if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) || + // !llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) { + // fprintf(stderr, "%s : partial rollback failed\n", __func__); + // return 1; + //} - constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY; - common_prompt_checkpoint ckpt_partial; - ckpt_partial.update_tgt(ctx_src, 0, partial_flags); - ckpt_partial.load_tgt(ctx_dst, 0, partial_flags); + //constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY; + //common_prompt_checkpoint ckpt_partial; + //ckpt_partial.update_tgt(ctx_src, 0, partial_flags); + //ckpt_partial.load_tgt(ctx_dst, 0, partial_flags); - if (!replay_and_compare("partial")) { - return 1; - } + //if (!replay_and_compare("partial")) { + // return 1; + //} // Repeat the load into a context that already has its own rollback state: // groups 1..n_rs_seq hold a different prompt's history, and rs_idx[0] is // non-zero at load time. The restore must wipe that state and still match. - llama_context * ctx_dirty = make_ctx(params, model); + llama_context * ctx_dirty = make_ctx(params, model, fill); if (ctx_dirty == nullptr) { fprintf(stderr, "%s : failed to init dirty ctx\n", __func__); return 1; @@ -379,13 +456,18 @@ int main(int argc, char ** argv) { return 1; } + const double nmse_dirty = nmse(logits_src_replay[i].data(), logits_dirty, n_vocab); + int token_first = -1; for (int token = 0; token < n_vocab; ++token) { - if (std::fabs(logits_src_replay[i][token] - logits_dirty[token]) > eps) { - fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, token %d (%g != %g)\n", - __func__, pos, token, (double) logits_src_replay[i][token], (double) logits_dirty[token]); - return 1; + if (logit_diff(logits_src_replay[i][token], logits_dirty[token]) > 0.0f && token_first < 0) { + token_first = token; } } + if (nmse_dirty > nmse_eps) { + fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, first token %d, nmse %g\n", + __func__, pos, token_first, nmse_dirty); + return 1; + } } fprintf(stderr, "%s : recurrent rollback checkpoint restored successfully\n", __func__); @@ -393,9 +475,46 @@ int main(int argc, char ** argv) { llama_free(ctx_dst); llama_free(ctx_dirty); - if (!test_multi_seq_split_replay(params, model, n_vocab)) { + if (!test_multi_seq_split_replay(params, model, n_vocab, fill)) { return 1; } return 0; } + +int main(int argc, char ** argv) { + std::setlocale(LC_NUMERIC, "C"); + + common_params params; + params.sampling.seed = 1234; + params.n_predict = 1; + + common_init(); + + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + ggml_backend_load_all(); + + common_init_result_ptr llama_init = common_init_from_params(params); + llama_model * model = llama_init->model(); + if (model == nullptr) { + fprintf(stderr, "%s : failed to init model\n", __func__); + return 1; + } + + if (!llama_model_is_recurrent(model) && !llama_model_is_hybrid(model)) { + fprintf(stderr, "%s : skipping for non-recurrent model\n", __func__); + return 0; + } + + for (uint8_t fill : { 0, 0x3e }) { + fprintf(stderr, "%s : testing with cache fill 0x%02x\n", __func__, fill); + if (test_rollback(params, model, fill) != 0) { + return 1; + } + } + + return 0; +} diff --git a/tests/test-sampling.cpp b/tests/test-sampling.cpp index d727ab632afb..353a5a1a1df2 100644 --- a/tests/test-sampling.cpp +++ b/tests/test-sampling.cpp @@ -7,6 +7,7 @@ #include <algorithm> #include <cmath> +#include <cstdlib> #include <string> #include <vector> diff --git a/tests/test-save-load-state.cpp b/tests/test-save-load-state.cpp index 6179e6c10848..08c7c67727e8 100644 --- a/tests/test-save-load-state.cpp +++ b/tests/test-save-load-state.cpp @@ -11,6 +11,43 @@ #include <string> #include <vector> +constexpr double NMSE_THRESHOLD = 1e-5; + +// normalized mean squared error = mse(a, b) / mse(a, 0) +static double nmse(const std::vector<float> & a, const std::vector<float> & b) { + GGML_ASSERT(a.size() == b.size()); + double mse_a_b = 0.0; + double mse_a_0 = 0.0; + + for (size_t i = 0; i < a.size(); i++) { + const float a_i = a[i]; + const float b_i = b[i]; + + mse_a_b += (double) (a_i - b_i) * (a_i - b_i); + mse_a_0 += (double) a_i * a_i; + } + + return mse_a_b / mse_a_0; +} + +struct generation_result { + llama_tokens tokens; + std::vector<std::vector<float>> logits; + + bool empty() const { return tokens.empty(); } +}; + +static bool get_current_logits(llama_context * ctx, std::vector<float> & out) { + const auto * vocab = llama_model_get_vocab(llama_get_model(ctx)); + const int32_t n_vocab = llama_vocab_n_tokens(vocab); + const float * logits = llama_get_logits_ith(ctx, -1); + if (logits == nullptr) { + return false; + } + out.assign(logits, logits + n_vocab); + return true; +} + struct llama_batch_ptr { llama_batch batch; @@ -28,15 +65,22 @@ struct llama_batch_ptr { const llama_batch & get() const { return batch; } }; -static llama_tokens generate_tokens(llama_context * ctx, llama_sampler * smpl, int & n_past, int32_t n_predict, llama_seq_id seq_id) { - llama_tokens result; +static generation_result generate_tokens(llama_context * ctx, llama_sampler * smpl, int & n_past, int32_t n_predict, llama_seq_id seq_id) { + generation_result result; llama_batch_ptr batch(1, 0, 1); for (int i = 0; i < n_predict; i++) { + std::vector<float> logits; + if (!get_current_logits(ctx, logits)) { + LOG_ERR("\n%s: failed to get logits\n", __func__); + return {}; + } + auto next_token = llama_sampler_sample(smpl, ctx, -1); LOG("%d ", next_token); - result.push_back(next_token); + result.tokens.push_back(next_token); + result.logits.push_back(std::move(logits)); common_batch_clear(batch.get()); common_batch_add(batch.get(), next_token, n_past, {seq_id}, true); @@ -51,12 +95,61 @@ static llama_tokens generate_tokens(llama_context * ctx, llama_sampler * smpl, i return result; } +static bool generate_tokens_compare( + llama_context * ctx, llama_sampler * smpl, int & n_past, int32_t n_predict, llama_seq_id seq_id, + const generation_result & expected) { + if (expected.tokens.size() != expected.logits.size() || expected.tokens.size() < (size_t) n_predict) { + LOG_ERR("\n%s: invalid expected generation\n", __func__); + return false; + } + + llama_batch_ptr batch(1, 0, 1); + + for (int i = 0; i < n_predict; i++) { + std::vector<float> logits; + if (!get_current_logits(ctx, logits)) { + LOG_ERR("\n%s: failed to get logits\n", __func__); + return false; + } + if (logits.size() != expected.logits[i].size()) { + LOG_ERR("\n%s: logits size mismatch at step %d: %zu != %zu\n", __func__, i, logits.size(), expected.logits[i].size()); + return false; + } + + const double nmse_val = nmse(expected.logits[i], logits); + LOG_TRC("%s: step %d nmse = %.6e\n", __func__, i, nmse_val); + if (nmse_val > NMSE_THRESHOLD) { + LOG_ERR("\n%s: error: NMSE at step %d is %.6e (threshold %.1e)\n", __func__, i, nmse_val, NMSE_THRESHOLD); + return false; + } + + const auto next_token = llama_sampler_sample(smpl, ctx, -1); + const auto expected_token = expected.tokens[i]; + + LOG("%d ", next_token); + if (next_token != expected_token) { + LOG_TRC("%s: sampled token %d differs from expected %d, using expected token\n", __func__, next_token, expected_token); + } + + common_batch_clear(batch.get()); + common_batch_add(batch.get(), expected_token, n_past, {seq_id}, true); + + if (llama_decode(ctx, batch.get())) { + LOG_ERR("\n%s: failed to evaluate\n", __func__); + return false; + } + n_past++; + } + + return true; +} + // Test 1: baseline // - decode all but the last token // - save state to disk // - decode the last token // - generate n_predict tokens -static llama_tokens test_baseline(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens) { +static generation_result test_baseline(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens) { auto params_ctx = common_context_params_to_llama(params); params_ctx.n_seq_max = 2; auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)}; @@ -109,7 +202,7 @@ static bool test_seq_rm_isolated( for (llama_seq_id seq_id = 0; seq_id < 2; ++seq_id) { llama_batch_ptr batch(n_tokens, 0, 1); for (size_t i = 0; i < n_tokens; ++i) { - common_batch_add(batch.get(), tokens[i], i, { seq_id }, false); + common_batch_add(batch.get(), tokens[i], i, { seq_id }, i == n_tokens - 1); } if (llama_decode(ctx.get(), batch.get())) { @@ -166,7 +259,7 @@ static bool test_seq_rm_isolated( // - load state from file // - replay the last prompt token // - generate n_predict tokens and compare against expected result -static bool test_state_load(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens, const llama_tokens & expected_result) { +static bool test_state_load(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens, const generation_result & expected_result) { auto params_ctx = common_context_params_to_llama(params); params_ctx.n_seq_max = 2; auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)}; @@ -195,14 +288,8 @@ static bool test_state_load(struct llama_model * model, const struct common_para } n_past++; - // Generate tokens - auto result = generate_tokens(ctx.get(), smpl.get(), n_past, params.n_predict, 0); - if (result.empty()) { - return false; - } - - if (result != expected_result) { - LOG_ERR("\n%s: error: generation differs from expected\n", __func__); + // Generate tokens and compare logits against the baseline + if (!generate_tokens_compare(ctx.get(), smpl.get(), n_past, params.n_predict, 0, expected_result)) { return false; } @@ -217,7 +304,7 @@ static bool test_state_load(struct llama_model * model, const struct common_para // - replay the last prompt token // - migrate KV cache from seq 0 to seq 1 via the CPU path // - generate n_predict tokens on seq 1 and compare against expected result -static bool test_seq_cp_host(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens, const llama_tokens & expected_result) { +static bool test_seq_cp_host(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens, const generation_result & expected_result) { auto params_ctx = common_context_params_to_llama(params); params_ctx.n_seq_max = 2; auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)}; @@ -267,14 +354,8 @@ static bool test_seq_cp_host(struct llama_model * model, const struct common_par LOG_TRC("%s: seq 1 restored, %zd bytes\n", __func__, nset); } - // Generate tokens on seq 1 - auto result = generate_tokens(ctx.get(), smpl.get(), n_past, params.n_predict, 1); - if (result.empty()) { - return false; - } - - if (result != expected_result) { - LOG_ERR("\n%s: error: generation differs from expected\n", __func__); + // Generate tokens and compare logits against the baseline + if (!generate_tokens_compare(ctx.get(), smpl.get(), n_past, params.n_predict, 1, expected_result)) { return false; } @@ -289,7 +370,7 @@ static bool test_seq_cp_host(struct llama_model * model, const struct common_par // - replay the last prompt token // - migrate KV cache from seq 0 to seq 1 via the on-device path // - generate n_predict tokens on seq 1 and compare against expected result -static bool test_seq_cp_device(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens, const llama_tokens & expected_result) { +static bool test_seq_cp_device(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens, const generation_result & expected_result) { auto params_ctx = common_context_params_to_llama(params); params_ctx.n_seq_max = 2; auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)}; @@ -339,14 +420,8 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_p LOG_TRC("%s: seq 1 restored, %zd bytes\n", __func__, nset); } - // Generate tokens on seq 1 - auto result = generate_tokens(ctx.get(), smpl.get(), n_past, params.n_predict, 1); - if (result.empty()) { - return false; - } - - if (result != expected_result) { - LOG_ERR("\n%s: error: generation differs from expected\n", __func__); + // Generate tokens and compare logits against the baseline + if (!generate_tokens_compare(ctx.get(), smpl.get(), n_past, params.n_predict, 1, expected_result)) { return false; } @@ -373,7 +448,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_ auto decode_one = [&](llama_token tok, int pos, llama_seq_id seq) { llama_batch_ptr batch(1, 0, 1); - common_batch_add(batch.get(), tok, pos, { seq }, false); + common_batch_add(batch.get(), tok, pos, { seq }, true); return llama_decode(ctx.get(), batch.get()) == 0; }; diff --git a/tools/cli/README.md b/tools/cli/README.md index b874d020730d..acecc74a6d9d 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -55,9 +55,6 @@ | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) | | `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) | -| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) | -| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) | -| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) | | `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) | | `-lzm, --lazy-mode MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)<br/>- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)<br/>- auto: on, but only for tensors larger than 4 GiB<br/>- off: always keep them resident<br/>(env: LLAMA_ARG_LAZY_MODE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) | @@ -90,6 +87,7 @@ | `-hft, --hf-token TOKEN` | Hugging Face access token (default: value from HF_TOKEN environment variable)<br/>(env: HF_TOKEN) | | `--log-disable` | Log disable | | `--log-file FNAME` | Log to file<br/>(env: LLAMA_ARG_LOG_FILE) | +| `--log-jsonl, --no-log-jsonl` | Log as JSONL (one JSON object per line) to stdout, this also disables colored logging (default: disabled)<br/>(env: LLAMA_ARG_LOG_JSONL) | | `--log-colors [on\|off\|auto]` | Set colored logging ('on', 'off', or 'auto', default: 'auto')<br/>'auto' enables colors when output is to a terminal<br/>(env: LLAMA_ARG_LOG_COLORS) | | `-v, --verbose, --log-verbose` | Set verbosity level to infinity (i.e. log all messages, useful for debugging) | | `--offline` | Offline mode: forces use of cache, prevents network access<br/>(env: LLAMA_ARG_OFFLINE) | @@ -108,18 +106,18 @@ | `-s, --seed SEED` | RNG seed (default: -1, use random seed for -1) | | `--sampler-seq, --sampling-seq SEQUENCE` | simplified sequence for samplers that will be used (default: edskypmxt) | | `--ignore-eos` | ignore end of stream token and continue generating (implies --logit-bias EOS-inf) | -| `--temp, --temperature N` | temperature (default: 0.80) | +| `--temp, --temperature N` | temperature (default: 0.80)<br/>(env: LLAMA_ARG_TEMPERATURE) | | `--top-k N` | top-k sampling (default: 40, 0 = disabled)<br/>(env: LLAMA_ARG_TOP_K) | -| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled) | -| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled) | +| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled)<br/>(env: LLAMA_ARG_TOP_P) | +| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled)<br/>(env: LLAMA_ARG_MIN_P) | | `--top-nsigma, --top-n-sigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) | | `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) | | `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) | | `--typical, --typical-p N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) | | `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled) | -| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) | -| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) | -| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled) | +| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled)<br/>(env: LLAMA_ARG_REPEAT_PENALTY) | +| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled)<br/>(env: LLAMA_ARG_PRESENCE_PENALTY) | +| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled)<br/>(env: LLAMA_ARG_FREQUENCY_PENALTY) | | `--dry-multiplier N` | set DRY sampling multiplier (default: 0.00, 0.0 = disabled) | | `--dry-base N` | set DRY sampling base value (default: 1.75) | | `--dry-allowed-length N` | set allowed length for DRY sampling (default: 2) | @@ -135,8 +133,8 @@ | `-l, --logit-bias TOKEN_ID(+/-)BIAS` | modifies the likelihood of token appearing in the completion,<br/>i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',<br/>or `--logit-bias 15043-1` to decrease likelihood of token ' Hello' | | `--grammar GRAMMAR` | BNF-like grammar to constrain generations (see samples in grammars/ dir) | | `--grammar-file FNAME` | file to read grammar from | -| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object<br/>For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | -| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object<br/>For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | +| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | +| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | | `-bs, --backend-sampling` | enable backend sampling (experimental) (default: disabled)<br/>(env: LLAMA_ARG_BACKEND_SAMPLING) | @@ -163,7 +161,7 @@ | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md<br/>(env: LLAMA_ARG_MMPROJ_URL) | | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_AUTO) | | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_OFFLOAD) | -| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)<br/>use --list-devices to see a list of available devices<br/>(env: MTMD_BACKEND_DEVICE) | +| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: follows --device)<br/>use --list-devices to see a list of available devices<br/>(env: MTMD_BACKEND_DEVICE) | | `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | @@ -178,7 +176,7 @@ | `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,<br/>or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)<br/>(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)<br/>(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)<br/>(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | -| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) | +| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) | | `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) | | `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) | | `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)<br/>(env: LLAMA_ARG_SKIP_CHAT_PARSING) | @@ -206,7 +204,7 @@ | `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) | | `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) | | `--spec-draft-backend-sampling, --no-spec-draft-backend-sampling` | offload draft sampling to the backend (default: enabled)<br/>(env: LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING) | -| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices | +| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload, default: follows --device)<br/>use --list-devices to see a list of available devices | | `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) | | `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_MODEL) | | `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,draft-dspark,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) | diff --git a/tools/completion/README.md b/tools/completion/README.md index 145be77e31c9..e2ac0668c647 100644 --- a/tools/completion/README.md +++ b/tools/completion/README.md @@ -138,9 +138,6 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) | | `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) | -| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) | -| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) | -| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) | | `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) | | `-lzm, --lazy-mode MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)<br/>- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)<br/>- auto: on, but only for tensors larger than 4 GiB<br/>- off: always keep them resident<br/>(env: LLAMA_ARG_LAZY_MODE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) | @@ -173,6 +170,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-hft, --hf-token TOKEN` | Hugging Face access token (default: value from HF_TOKEN environment variable)<br/>(env: HF_TOKEN) | | `--log-disable` | Log disable | | `--log-file FNAME` | Log to file<br/>(env: LLAMA_ARG_LOG_FILE) | +| `--log-jsonl, --no-log-jsonl` | Log as JSONL (one JSON object per line) to stdout, this also disables colored logging (default: disabled)<br/>(env: LLAMA_ARG_LOG_JSONL) | | `--log-colors [on\|off\|auto]` | Set colored logging ('on', 'off', or 'auto', default: 'auto')<br/>'auto' enables colors when output is to a terminal<br/>(env: LLAMA_ARG_LOG_COLORS) | | `-v, --verbose, --log-verbose` | Set verbosity level to infinity (i.e. log all messages, useful for debugging) | | `--offline` | Offline mode: forces use of cache, prevents network access<br/>(env: LLAMA_ARG_OFFLINE) | @@ -191,18 +189,18 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-s, --seed SEED` | RNG seed (default: -1, use random seed for -1) | | `--sampler-seq, --sampling-seq SEQUENCE` | simplified sequence for samplers that will be used (default: edskypmxt) | | `--ignore-eos` | ignore end of stream token and continue generating (implies --logit-bias EOS-inf) | -| `--temp, --temperature N` | temperature (default: 0.80) | +| `--temp, --temperature N` | temperature (default: 0.80)<br/>(env: LLAMA_ARG_TEMPERATURE) | | `--top-k N` | top-k sampling (default: 40, 0 = disabled)<br/>(env: LLAMA_ARG_TOP_K) | -| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled) | -| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled) | +| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled)<br/>(env: LLAMA_ARG_TOP_P) | +| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled)<br/>(env: LLAMA_ARG_MIN_P) | | `--top-nsigma, --top-n-sigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) | | `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) | | `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) | | `--typical, --typical-p N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) | | `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled) | -| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) | -| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) | -| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled) | +| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled)<br/>(env: LLAMA_ARG_REPEAT_PENALTY) | +| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled)<br/>(env: LLAMA_ARG_PRESENCE_PENALTY) | +| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled)<br/>(env: LLAMA_ARG_FREQUENCY_PENALTY) | | `--dry-multiplier N` | set DRY sampling multiplier (default: 0.00, 0.0 = disabled) | | `--dry-base N` | set DRY sampling base value (default: 1.75) | | `--dry-allowed-length N` | set allowed length for DRY sampling (default: 2) | @@ -218,8 +216,8 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-l, --logit-bias TOKEN_ID(+/-)BIAS` | modifies the likelihood of token appearing in the completion,<br/>i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',<br/>or `--logit-bias 15043-1` to decrease likelihood of token ' Hello' | | `--grammar GRAMMAR` | BNF-like grammar to constrain generations (see samples in grammars/ dir) | | `--grammar-file FNAME` | file to read grammar from | -| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object<br/>For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | -| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object<br/>For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | +| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | +| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | | `-bs, --backend-sampling` | enable backend sampling (experimental) (default: disabled)<br/>(env: LLAMA_ARG_BACKEND_SAMPLING) | @@ -256,7 +254,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,<br/>or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)<br/>(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)<br/>(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)<br/>(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | -| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) | +| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) | | `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) | | `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) | | `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)<br/>(env: LLAMA_ARG_SKIP_CHAT_PARSING) | @@ -558,7 +556,7 @@ These options help improve the performance and memory usage of the LLaMA models. - `--grammar GRAMMAR`, `--grammar-file FILE`: Specify a grammar (defined inline or in a file) to constrain model output to a specific format. For example, you could force the model to output JSON or to speak only in emojis. See the [GBNF guide](../../grammars/README.md) for details on the syntax. -- `--json-schema SCHEMA`: Specify a [JSON schema](https://json-schema.org/) to constrain model output to (e.g. `{}` for any JSON object, or `{"items": {"type": "string", "minLength": 10, "maxLength": 100}, "minItems": 10}` for a JSON array of strings with size constraints). If a schema uses external `$ref`s, you should use `--grammar "$( python examples/json_schema_to_grammar.py myschema.json )"` instead. +- `--json-schema SCHEMA`: Specify a [JSON schema](https://json-schema.org/) to constrain model output to (e.g. `{"type": "object"}` for any JSON object, or `{"items": {"type": "string", "minLength": 10, "maxLength": 100}, "minItems": 10}` for a JSON array of strings with size constraints). ### Quantization diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index 5faae8a3bd0c..8e3d3072fbe4 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -429,6 +429,7 @@ static void print_usage(int /* argc */, char ** argv) { printf("\n"); printf("options:\n"); printf(" -h, --help\n"); + printf(" --version show version and build info\n"); printf(" --numa <distribute|isolate|numactl> numa mode (default: disabled)\n"); printf(" -r, --repetitions <n> number of times to repeat each test (default: %d)\n", cmd_params_defaults.reps); printf(" --prio <-1|0|1|2|3> process/thread priority (default: %d)\n", cmd_params_defaults.prio); @@ -479,8 +480,6 @@ static void print_usage(int /* argc */, char ** argv) { printf(" -dev, --device <dev0/dev1/...> (default: auto)\n"); printf(" -lm, --load-mode <auto|none|mmap|mlock|mmap+mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str()); printf(" -lzm, --lazy-mode <on|auto|off> (default: %s)\n", join(transform_to_str(cmd_params_defaults.lazy_mode, lazy_mode_str), ",").c_str()); - printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n"); - printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n"); printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str()); printf(" -ts, --tensor-split <ts0/ts1/..> (default: 0)\n"); printf(" -ot --override-tensor <tensor name pattern>=<buffer type>;...\n"); @@ -565,6 +564,9 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { if (arg == "-h" || arg == "--help") { print_usage(argc, argv); exit(0); + } else if (arg == "--version") { + llama_print_build_info(llama_version()); + exit(0); } else if (arg == "-m" || arg == "--model") { if (++i >= argc) { invalid_param = true; @@ -896,44 +898,6 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { break; } params.flash_attn.insert(params.flash_attn.end(), types.begin(), types.end()); - } else if (arg == "-mmp" || arg == "--mmap") { - if (++i >= argc) { - invalid_param = true; - break; - } - LOG_WRN("DEPRECATED: -mmp and --mmap are deprecated in favour of --load-mode. Please use --load-mode mmap instead.\n"); - auto p = string_split<bool>(argv[i], split_delim); - - std::vector<llama_load_mode> modes; - for (const auto & m : p) { - llama_load_mode mode; - if (m) { - mode = LLAMA_LOAD_MODE_MMAP; - } else { - mode = LLAMA_LOAD_MODE_NONE; - } - modes.push_back(mode); - } - params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end()); - } else if (arg == "-dio" || arg == "--direct-io") { - if (++i >= argc) { - invalid_param = true; - break; - } - LOG_WRN("DEPRECATED: -dio and --direct-io are deprecated in favour of --load-mode. Please use --load-mode dio instead.\n"); - auto p = string_split<bool>(argv[i], split_delim); - - std::vector<llama_load_mode> modes; - for (const auto & m : p) { - llama_load_mode mode; - if (m) { - mode = LLAMA_LOAD_MODE_DIRECT_IO; - } else { - mode = LLAMA_LOAD_MODE_NONE; - } - modes.push_back(mode); - } - params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end()); } else if (arg == "-embd" || arg == "--embeddings") { if (++i >= argc) { invalid_param = true; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 74f4e2b5a490..feceb7ff765a 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -780,7 +780,7 @@ ggml_tensor * clip_graph::build_attn( } cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f); - ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); if (sinks != nullptr) { ggml_flash_attn_ext_add_sinks(cur, sinks); } @@ -793,7 +793,7 @@ ggml_tensor * clip_graph::build_attn( ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); // F32 may not needed for vision encoders? - // ggml_mul_mat_set_prec(kq, GGML_PREC_F32); + // ggml_prec_set_acc(kq, GGML_PREC_F32); kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f); if (sinks != nullptr) { @@ -1408,9 +1408,14 @@ struct clip_model_loader { } // Load the vision/audio feature layer indices if they are explicitly provided - // NOTE: gguf conversions should standardize the values of the vision feature layer to - // be non-negative, since we use -1 to mark values as unset here. + // NOTE: gguf conversions should standardize the values of the vision feature layer to be non-negative, since we use -1 to mark values as unset here. get_arr_int(string_format(KEY_FEATURE_LAYERS, prefix), hparams.feature_layers, false); + for (const auto & v : hparams.feature_layers) { + if (v > (int) hparams.n_layer) { + throw std::runtime_error(string_format("%s: feature layer index %d is out of range (n_layer: %d)", + __func__, v, hparams.n_layer)); + } + } // model-specific params switch (model.proj_type) { @@ -1456,7 +1461,12 @@ struct clip_model_loader { std::vector<int> wa_layer_indexes_vec; get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, false); if (!wa_layer_indexes_vec.empty()) { - hparams.insert_layer_id = wa_layer_indexes_vec[0]; + const int insert_lid = wa_layer_indexes_vec[0]; + if (insert_lid < 0 || insert_lid >= (int) hparams.n_layer) { + throw std::runtime_error(string_format("%s: layer index %d is out of range (n_layer: %d)", + __func__, insert_lid, hparams.n_layer)); + } + hparams.insert_layer_id = insert_lid; } } break; case PROJECTOR_TYPE_INTERNVL: @@ -3226,6 +3236,7 @@ struct clip_model_loader { model.pos_embed = get_tensor(string_format(TN_SAM_POS_EMBD, "weight")); model.patch_embed_proj_w = get_tensor(string_format(TN_SAM_PATCH_EMBD, "weight")); model.patch_embed_proj_b = get_tensor(string_format(TN_SAM_PATCH_EMBD, "bias")); + model.n_sam_layers = hparams.sam_n_layer; model.sam_layers.resize(model.n_sam_layers); for (int il = 0; il < model.n_sam_layers; ++il) { auto & layer = model.sam_layers[il]; @@ -4436,7 +4447,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { // build the inference graph ggml_backend_sched_reset(ctx->sched.get()); ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs, params)->build(); - ggml_backend_sched_alloc_graph(ctx->sched.get(), gf); + if (!ggml_backend_sched_alloc_graph(ctx->sched.get(), gf)) { + LOG_ERR("%s: failed to allocate compute graph\n", __func__); + return false; + } // set inputs const auto & model = ctx->model; @@ -4649,8 +4663,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316 std::vector<int32_t> positions(pos_h * pos_w); - int bucket_coords_h[1024]; - int bucket_coords_w[1024]; + // note: sized by the actual patch counts; a tall/wide image produces more + // than 1024 patches per side and a fixed [1024] array would be overrun + std::vector<int> bucket_coords_h(pos_h); + std::vector<int> bucket_coords_w(pos_w); for (int i = 0; i < pos_h; i++){ bucket_coords_h[i] = std::floor(70.0*i/pos_h); } @@ -4693,8 +4709,8 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { // SigLIP position buckets (same as resampler path) std::vector<int32_t> positions(pos_h * pos_w); - int bucket_coords_h[1024]; - int bucket_coords_w[1024]; + std::vector<int> bucket_coords_h(pos_h); + std::vector<int> bucket_coords_w(pos_w); for (int i = 0; i < pos_h; i++){ bucket_coords_h[i] = std::floor(70.0*i/pos_h); } diff --git a/tools/mtmd/deprecation-warning.cpp b/tools/mtmd/deprecation-warning.cpp index 2b31a9d8b0b3..615d7577bca4 100644 --- a/tools/mtmd/deprecation-warning.cpp +++ b/tools/mtmd/deprecation-warning.cpp @@ -1,5 +1,6 @@ #include <clocale> #include <cstdio> +#include <cstdlib> #include <string> int main(int argc, char** argv) { diff --git a/tools/mtmd/models/mimovl.cpp b/tools/mtmd/models/mimovl.cpp index 6ff1124a02f3..e1fbe2671dcf 100644 --- a/tools/mtmd/models/mimovl.cpp +++ b/tools/mtmd/models/mimovl.cpp @@ -2,7 +2,7 @@ ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const { ggml_tensor * cur = ggml_mul_mat(ctx0, w, x); - ggml_mul_mat_set_prec(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_F32); return cur; } diff --git a/tools/mtmd/models/qwen3tts-spkenc.cpp b/tools/mtmd/models/qwen3tts-spkenc.cpp index d4659fd63dc4..405fbb9cbc20 100644 --- a/tools/mtmd/models/qwen3tts-spkenc.cpp +++ b/tools/mtmd/models/qwen3tts-spkenc.cpp @@ -27,7 +27,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens ggml_tensor * w2d = ggml_reshape_2d(ctx0, w, (int64_t) K * IC, OC); ggml_tensor * y = ggml_mul_mat(ctx0, w2d, col); // [OC, T_out] - ggml_mul_mat_set_prec(y, GGML_PREC_F32); + ggml_prec_set_acc(y, GGML_PREC_F32); ggml_tensor * b2d = ggml_reshape_2d(ctx0, b, OC, 1); y = ggml_add(ctx0, y, b2d); diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index dc2ab414e742..bdf8bf6fe450 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -371,6 +371,7 @@ static bool is_webp_file(const unsigned char * buf, size_t len) { #ifdef MTMD_VIDEO static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder, const mtmd_helper_video_init_params & params); +static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string & id); #endif mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder, @@ -436,6 +437,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * LOG_ERR("%s: failed to decode buffer as either image/audio/video\n", __func__); return {nullptr, nullptr}; } + mtmd_helper_video_set_id(video_ctx, id); // propagate the hash to the frames result = mtmd_bitmap_init_lazy(ctx, id.empty() ? nullptr : id.c_str(), video_ctx, @@ -527,6 +529,7 @@ struct mtmd_helper_video { std::string ffprobe_bin; float fps_target = 0.0f; mtmd_helper_video_info info = {}; + std::string id; // hash of the input video // RAII wrapper for managing subprocess struct subprocess_handle { @@ -785,9 +788,14 @@ struct mtmd_helper_video { } LOG_DBG("%s: frame %d read OK\n", __func__, current_frame); - current_frame++; mtmd_bitmap * frame = mtmd_bitmap_init(info.width, info.height, frame_buf.data()); mtmd_bitmap_set_mergeable(frame, true); + if (!id.empty()) { + // each frame gets a unique id in the form of {hash}+{frame}, so that it can be identified in cache + std::string frame_id = id + "+" + std::to_string(current_frame); + mtmd_bitmap_set_id(frame, frame_id.c_str()); + } + current_frame++; return frame; } @@ -886,6 +894,10 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) { } #ifdef MTMD_VIDEO +static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string & id) { + vctx->id = id; +} + static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder, const mtmd_helper_video_init_params & params) { mtmd_helper_video vctx; diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index c11d35c87d7c..9b2e98862df4 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -294,8 +294,8 @@ struct img_tool { support = filter_support * filterscale; // Widen filter when downsampling ksize = static_cast<int>(std::ceil(support)) * 2 + 1; // Total pixels in kernel - std::vector<double> pre_weights(outSize * ksize); // Temporary weights - bounds.resize(outSize * 2); + std::vector<double> pre_weights((size_t) outSize * ksize); // Temporary weights + bounds.resize((size_t) outSize * 2); // For each output pixel, compute its filter coefficients @@ -322,20 +322,20 @@ struct img_tool { for (x = 0; x < xmax; x++) { // Distance from input pixel center to output pixel center in input space double w = resample_filter((x + xmin - center + 0.5) * ss); - pre_weights[xx * ksize + x] = w; + pre_weights[(size_t) xx * ksize + x] = w; ww += w; // Accumulate for normalization } // Normalize weights to sum to 1.0 (preserves brightness) for (x = 0; x < xmax; x++) { if (ww != 0.0) { - pre_weights[xx * ksize + x] /= ww; + pre_weights[(size_t) xx * ksize + x] /= ww; } } // Zero-pad remaining kernel positions for (; x < ksize; x++) { - pre_weights[xx * ksize + x] = 0; + pre_weights[(size_t) xx * ksize + x] = 0; } // Store input pixel range for this output pixel @@ -345,11 +345,11 @@ struct img_tool { // Convert floating-point coefficients to fixed-point integers // Formula: int32 = round(float * 2^PRECISION_BITS) - weights.resize(outSize * ksize); + weights.resize((size_t) outSize * ksize); const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITS - for (int i = 0; i < outSize * ksize; i++) { + for (size_t i = 0; i < (size_t) outSize * ksize; i++) { // Pillow adds +/- 0.5 then truncates toward zero; std::round would round twice const double rounded = pre_weights[i] * fxp_scale + (pre_weights[i] < 0 ? -0.5 : 0.5); weights[i] = static_cast<int32_t>(rounded); @@ -442,6 +442,12 @@ struct img_tool { const int src_width = img.get_size().width; const int src_height = img.get_size().height; + // sanity check on the target size + if (target_width <= 0 || target_width > 65536 || target_height <= 0 || target_height > 65536) { + throw std::runtime_error("resize target " + std::to_string(target_width) + "x" + + std::to_string(target_height) + " is out of range (max 65536)"); + } + bool need_horizontal = (target_width != src_width); bool need_vertical = (target_height != src_height); diff --git a/tools/rpc/rpc-server.cpp b/tools/rpc/rpc-server.cpp index 08e680391415..2db5e52ab8c0 100644 --- a/tools/rpc/rpc-server.cpp +++ b/tools/rpc/rpc-server.cpp @@ -1,3 +1,4 @@ +#include "ggml-backend.h" #include "ggml-rpc.h" #ifdef _WIN32 # define NOMINMAX @@ -270,7 +271,8 @@ static std::vector<ggml_backend_dev_t> get_devices(const rpc_server_params & par if (devices.empty()) { for (size_t i = 0; i < ggml_backend_dev_count(); i++) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); - if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) { + enum ggml_backend_dev_type dev_type = ggml_backend_dev_type(dev); + if (dev_type != GGML_BACKEND_DEVICE_TYPE_CPU && dev_type != GGML_BACKEND_DEVICE_TYPE_ACCEL) { devices.push_back(dev); } } diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txt index 42e693a7078d..b693cdf3878c 100644 --- a/tools/server/CMakeLists.txt +++ b/tools/server/CMakeLists.txt @@ -30,8 +30,8 @@ if (BUILD_SHARED_LIBS) endif() target_include_directories(${TARGET} PRIVATE ../mtmd) -target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}) -target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src) +target_include_directories(${TARGET} PRIVATE ${PROJECT_SOURCE_DIR}) +target_include_directories(${TARGET} PRIVATE ${PROJECT_SOURCE_DIR}/src) if(GGML_HIP) target_include_directories(${TARGET} PRIVATE /opt/rocm/include) target_compile_definitions(${TARGET} PRIVATE __HIP_PLATFORM_AMD__) @@ -42,7 +42,7 @@ target_link_libraries(${TARGET} PUBLIC llama-common llama-expert-dispatch mtmd $ # No GPU, no model load -- see tools/server/tests-cpp/test-prefix-cache-match.cpp. if (LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION) add_executable(test-prefix-cache-match tests-cpp/test-prefix-cache-match.cpp) - target_include_directories(test-prefix-cache-match PRIVATE ${CMAKE_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/src ../mtmd) + target_include_directories(test-prefix-cache-match PRIVATE ${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/src ../mtmd) target_link_libraries(test-prefix-cache-match PRIVATE server-context) add_test(NAME test-prefix-cache-match COMMAND test-prefix-cache-match) endif() @@ -61,9 +61,11 @@ add_library(${TARGET} set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_include_directories(${TARGET} PRIVATE ../mtmd ${CMAKE_SOURCE_DIR} ../wp-expert-worker) +target_include_directories(${TARGET} PRIVATE ../mtmd ${PROJECT_SOURCE_DIR} ../wp-expert-worker) target_link_libraries(${TARGET} PUBLIC server-context llama-ui cpp-httplib llama-wp-expert-worker-lib ${CMAKE_THREAD_LIBS_INIT}) +add_dependencies(${TARGET} llama-ui-assets) + if(LLAMA_TOOLS_INSTALL) install(TARGETS ${TARGET} LIBRARY) endif() diff --git a/tools/server/README.md b/tools/server/README.md index c6e907ba9199..e665904ba2c1 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -72,9 +72,6 @@ For the full list of features, please refer to [server's changelog](https://gith | `-ctv, --cache-type-v TYPE` | KV cache data type for V<br/>allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1<br/>(default: f16)<br/>(env: LLAMA_ARG_CACHE_TYPE_V) | | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) | -| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) | -| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) | -| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) | | `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) | | `-lzm, --lazy-mode MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)<br/>- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)<br/>- auto: on, but only for tensors larger than 4 GiB<br/>- off: always keep them resident<br/>(env: LLAMA_ARG_LAZY_MODE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) | @@ -107,6 +104,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `-hft, --hf-token TOKEN` | Hugging Face access token (default: value from HF_TOKEN environment variable)<br/>(env: HF_TOKEN) | | `--log-disable` | Log disable | | `--log-file FNAME` | Log to file<br/>(env: LLAMA_ARG_LOG_FILE) | +| `--log-jsonl, --no-log-jsonl` | Log as JSONL (one JSON object per line) to stdout, this also disables colored logging (default: disabled)<br/>(env: LLAMA_ARG_LOG_JSONL) | | `--log-colors [on\|off\|auto]` | Set colored logging ('on', 'off', or 'auto', default: 'auto')<br/>'auto' enables colors when output is to a terminal<br/>(env: LLAMA_ARG_LOG_COLORS) | | `-v, --verbose, --log-verbose` | Set verbosity level to infinity (i.e. log all messages, useful for debugging) | | `--offline` | Offline mode: forces use of cache, prevents network access<br/>(env: LLAMA_ARG_OFFLINE) | @@ -125,18 +123,18 @@ For the full list of features, please refer to [server's changelog](https://gith | `-s, --seed SEED` | RNG seed (default: -1, use random seed for -1) | | `--sampler-seq, --sampling-seq SEQUENCE` | simplified sequence for samplers that will be used (default: edskypmxt) | | `--ignore-eos` | ignore end of stream token and continue generating (implies --logit-bias EOS-inf) | -| `--temp, --temperature N` | temperature (default: 0.80) | +| `--temp, --temperature N` | temperature (default: 0.80)<br/>(env: LLAMA_ARG_TEMPERATURE) | | `--top-k N` | top-k sampling (default: 40, 0 = disabled)<br/>(env: LLAMA_ARG_TOP_K) | -| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled) | -| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled) | +| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled)<br/>(env: LLAMA_ARG_TOP_P) | +| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled)<br/>(env: LLAMA_ARG_MIN_P) | | `--top-nsigma, --top-n-sigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) | | `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) | | `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) | | `--typical, --typical-p N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) | | `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled) | -| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) | -| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) | -| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled) | +| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled)<br/>(env: LLAMA_ARG_REPEAT_PENALTY) | +| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled)<br/>(env: LLAMA_ARG_PRESENCE_PENALTY) | +| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled)<br/>(env: LLAMA_ARG_FREQUENCY_PENALTY) | | `--dry-multiplier N` | set DRY sampling multiplier (default: 0.00, 0.0 = disabled) | | `--dry-base N` | set DRY sampling base value (default: 1.75) | | `--dry-allowed-length N` | set allowed length for DRY sampling (default: 2) | @@ -152,8 +150,8 @@ For the full list of features, please refer to [server's changelog](https://gith | `-l, --logit-bias TOKEN_ID(+/-)BIAS` | modifies the likelihood of token appearing in the completion,<br/>i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',<br/>or `--logit-bias 15043-1` to decrease likelihood of token ' Hello' | | `--grammar GRAMMAR` | BNF-like grammar to constrain generations (see samples in grammars/ dir) | | `--grammar-file FNAME` | file to read grammar from | -| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object<br/>For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | -| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object<br/>For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | +| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | +| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | | `-bs, --backend-sampling` | enable backend sampling (experimental) (default: disabled)<br/>(env: LLAMA_ARG_BACKEND_SAMPLING) | @@ -181,7 +179,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md<br/>(env: LLAMA_ARG_MMPROJ_URL) | | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_AUTO) | | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_OFFLOAD) | -| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)<br/>use --list-devices to see a list of available devices<br/>(env: MTMD_BACKEND_DEVICE) | +| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: follows --device)<br/>use --list-devices to see a list of available devices<br/>(env: MTMD_BACKEND_DEVICE) | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | | `--mtmd-batch-max-tokens N` | maximum number of image tokens per batch when encoding images (default: 1024)<br/>(env: LLAMA_ARG_MTMD_BATCH_MAX_TOKENS) | @@ -191,7 +189,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `-a, --alias STRING` | set model name aliases, comma-separated (to be used by API)<br/>(env: LLAMA_ARG_ALIAS) | | `--tags STRING` | set model tags, comma-separated (informational, not used for routing)<br/>(env: LLAMA_ARG_TAGS) | | `--embd-normalize N` | normalisation for embeddings (default: 2) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm) | -| `--host HOST` | ip address to listen, or bind to an UNIX socket if the address ends with .sock (default: 127.0.0.1)<br/>(env: LLAMA_ARG_HOST) | +| `--host HOST` | IP addresses to listen on, comma-separated, or UNIX socket paths ending in .sock; with multiple TCP addresses, :: binds IPv6 only; overlapping addresses result in undefined behavior (default: 127.0.0.1)<br/>(env: LLAMA_ARG_HOST) | | `--port PORT` | port to listen (default: 8080)<br/>(env: LLAMA_ARG_PORT) | | `--reuse-port` | allow multiple sockets to bind to the same port (default: disabled)<br/>(env: LLAMA_ARG_REUSE_PORT) | | `--path PATH` | path to serve static files from (default: )<br/>(env: LLAMA_ARG_STATIC_PATH) | @@ -236,7 +234,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,<br/>or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)<br/>(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)<br/>(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)<br/>(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | -| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) | +| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) | | `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) | | `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) | | `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)<br/>(env: LLAMA_ARG_SKIP_CHAT_PARSING) | @@ -267,7 +265,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) | | `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) | | `--spec-draft-backend-sampling, --no-spec-draft-backend-sampling` | offload draft sampling to the backend (default: enabled)<br/>(env: LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING) | -| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices | +| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload, default: follows --device)<br/>use --list-devices to see a list of available devices | | `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) | | `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_MODEL) | | `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,draft-dspark,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) | diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index a6fe3c6ba619..a3a4ea152fb3 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -203,10 +203,24 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { } else { json chatcmpl_outputs = item.at("output"); for (json & chatcmpl_output : chatcmpl_outputs) { - if (!chatcmpl_output.contains("type") || chatcmpl_output.at("type") != "input_text") { - throw std::invalid_argument("Output of tool call should be 'Input text'"); + if (!chatcmpl_output.contains("type")) { + throw std::invalid_argument("Output of tool call missing 'type' field"); + } + const auto type = chatcmpl_output.at("type"); + if (type != "input_text" && type != "input_image") { + throw std::invalid_argument("Output of tool call should be 'Input text' or 'Input image'"); + } + if (type == "input_text") { + chatcmpl_output["type"] = "text"; + } else if (type == "input_image") { + if (!chatcmpl_output.contains("image_url")) { + throw std::invalid_argument("'image_url' is required"); + } + chatcmpl_output["type"] = "image_url"; + chatcmpl_output["image_url"] = json { + {"url", chatcmpl_output.at("image_url")} + }; } - chatcmpl_output["type"] = "text"; } chatcmpl_messages.push_back(json { {"content", chatcmpl_outputs}, diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index d38cd65dad25..5672193cff78 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -15,6 +15,23 @@ #include <limits> #include <cstring> #include <type_traits> +#include <chrono> +#include <thread> + +#ifdef _WIN32 +// windows.h defines min and max as macros, which breaks std::min and std::max +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include <windows.h> +#include <io.h> +#else +#include <errno.h> +#include <fcntl.h> +#include <poll.h> +#include <unistd.h> +#endif json format_error_response(const std::string & message, const enum error_type type) { std::string type_str; @@ -1181,6 +1198,11 @@ json oaicompat_chat_params_parse( } } + // an absent or empty schema means any object + if (json_schema.is_object() && json_schema.empty()) { + json_schema["type"] = "object"; + } + // get input files if (!body.contains("messages")) { throw std::invalid_argument("'messages' is required"); @@ -1241,12 +1263,13 @@ json oaicompat_chat_params_parse( p["text"] = get_media_marker(); p.erase("input_audio"); - } else if (type == "input_video") { + } else if (type == "input_video" || type == "video_url") { if (!opt.allow_video) { throw std::runtime_error("video input is not supported - hint: if this is unexpected, you may need to provide the mmproj"); } - json input_video = json_value(p, "input_video", json::object()); + // accept the OpenAI-style "video_url" key as an alias of "input_video" + json input_video = json_value(p, type, json::object()); std::string url = json_value(input_video, "data", json_value(input_video, "url", std::string())); handle_media(out_files, url, opt.media_path); @@ -1254,6 +1277,7 @@ json oaicompat_chat_params_parse( p["type"] = "media_marker"; p["text"] = get_media_marker(); p.erase("input_video"); + p.erase("video_url"); } else if (type != "text") { throw std::invalid_argument("unsupported content[].type"); @@ -1837,3 +1861,133 @@ server_tokens format_prompt_rerank( return result; } + +// +// server_subproc +// + +bool server_subproc::has_output() { + if (out_handle >= 0) { + return true; + } + FILE * f = sproc.stdout_file(); // combined stdout/stderr + if (!f) { + return false; + } +#ifdef _WIN32 + HANDLE h = (HANDLE) _get_osfhandle(_fileno(f)); + if (h != INVALID_HANDLE_VALUE) { + out_handle = (intptr_t) h; + } +#else + int fd = fileno(f); + if (fd >= 0) { + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + out_handle = fd; + } +#endif + return out_handle >= 0; +} + +int server_subproc::read_output(char * buf, size_t len) { + if (!has_output()) { + return -1; + } +#ifdef _WIN32 + HANDLE h = (HANDLE) out_handle; + DWORD avail = 0; + if (!PeekNamedPipe(h, NULL, 0, NULL, &avail, NULL)) { + return -1; // pipe broken, child gone + } + if (avail == 0) { + return 0; + } + DWORD to_read = avail < (DWORD) len ? avail : (DWORD) len; + DWORD got = 0; + if (!ReadFile(h, buf, to_read, &got, NULL) || got == 0) { + return -1; + } + return (int) got; +#else + while (true) { + ssize_t r = read((int) out_handle, buf, len); + if (r > 0) { + return (int) r; + } + if (r == 0) { + return -1; // EOF + } + if (errno == EINTR) { + continue; + } + if (errno == EAGAIN || errno == EWOULDBLOCK) { + return 0; + } + return -1; + } +#endif +} + +server_subproc::waiter::waiter() { +#ifndef _WIN32 + int fds[2]; + GGML_ASSERT(pipe(fds) == 0); + for (int fd : fds) { + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + } + wake_fd[0] = fds[0]; + wake_fd[1] = fds[1]; +#endif +} + +server_subproc::waiter::~waiter() { +#ifndef _WIN32 + close((int) wake_fd[0]); + close((int) wake_fd[1]); +#endif +} + +void server_subproc::waiter::wake() { +#ifndef _WIN32 + char c = 1; + (void) !write((int) wake_fd[1], &c, 1); +#endif +} + +void server_subproc::waiter::wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms) { + ready.assign(procs.size(), false); +#ifdef _WIN32 + // no waitable wait exists for anonymous pipes, so poll them in 50 ms steps + bool any = false; + for (size_t i = 0; i < procs.size(); i++) { + DWORD avail = 0; + if (!procs[i]->has_output() || !PeekNamedPipe((HANDLE) procs[i]->out_handle, NULL, 0, NULL, &avail, NULL) || avail > 0) { + ready[i] = true; // data or broken pipe, read_output() tells which + any = true; + } + } + if (!any) { + int64_t step = timeout_ms < 0 ? 50 : std::min<int64_t>(timeout_ms, 50); + std::this_thread::sleep_for(std::chrono::milliseconds(step)); + } +#else + std::vector<pollfd> pfds; + pfds.reserve(procs.size() + 1); + pfds.push_back({ (int) wake_fd[0], POLLIN, 0 }); + for (auto * p : procs) { + pfds.push_back({ p->has_output() ? (int) p->out_handle : -1, POLLIN, 0 }); // poll() skips negative fds + } + int timeout = timeout_ms < 0 ? -1 : (int) std::min<int64_t>(timeout_ms, std::numeric_limits<int>::max()); + int r = poll(pfds.data(), pfds.size(), timeout); + if (r < 0 && errno != EINTR) { + LOG_ERR("%s: poll() failed: %s\n", __func__, strerror(errno)); + } + if (pfds[0].revents) { + char buf[64]; + while (read((int) wake_fd[0], buf, sizeof(buf)) > 0) {} + } + for (size_t i = 0; i < procs.size(); i++) { + ready[i] = pfds[i + 1].fd < 0 || pfds[i + 1].revents != 0; + } +#endif +} diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 3b71aad2e10d..339f5dc439d8 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -6,6 +6,7 @@ #include "chat.h" #include "mtmd.h" #include "mtmd-helper.h" +#include "subproc.h" #include "json.h" @@ -13,6 +14,7 @@ #include <chrono> #include <condition_variable> #include <cinttypes> +#include <cstdio> #include <functional> #include <mutex> #include <queue> @@ -647,3 +649,39 @@ struct server_pipe { return true; } }; + +// wrapper around common_subproc to manage a child server process +// mainly used by router mode +struct server_subproc { + common_subproc sproc; + std::atomic<bool> stopped{false}; // set by the monitor once the process exited and was reaped + + bool is_alive() { return sproc.alive(); } + void terminate() { sproc.terminate(); } + int join() { return sproc.join(); } + + // true if the child's combined stdout/stderr pipe is available (call after create()) + bool has_output(); + + // non-blocking read + // returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken + int read_output(char * buf, size_t len); + + // wait until one of a set of children has output, wake() is called, or a timeout passes + struct waiter { + waiter(); + ~waiter(); + + // thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway + void wake(); + + // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data (or a broken pipe) + void wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms); + + private: + intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe + }; + +private: + intptr_t out_handle = -1; // fd on POSIX, HANDLE on Windows; taken lazily from sproc +}; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 03a8f3a68dcc..133381c8da23 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3966,8 +3966,11 @@ struct server_context_impl { // evict checkpoints within min-step of a previous checkpoint, unless they were // created by the current task + // only when the list is full, otherwise short prompts keep just the oldest checkpoint int64_t last = -1; - for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end(); ) { + for (auto it = slot.prompt.checkpoints.begin(); + slot.prompt.checkpoints.size() + 1 >= (size_t) params_base.n_ctx_checkpoints && + it != slot.prompt.checkpoints.end(); ) { if (it->id_task != id_task && last >= 0 && it->n_tokens <= last + params_base.checkpoint_min_step) { SLT_TRC(slot, "erasing context checkpoint too close to an earlier one (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, (float) it->size() / 1024 / 1024); @@ -3990,6 +3993,19 @@ struct server_context_impl { slot.prompt.checkpoints.erase(slot.prompt.checkpoints.begin()); } + // replace an existing checkpoint at the same n_tokens instead of appending a duplicate + { + const int64_t n_tokens_new = slot.prompt.n_tokens() - n_tokens_cur; + for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end(); ) { + if (it->n_tokens == n_tokens_new) { + SLT_TRC(slot, "superseding context checkpoint at n_tokens = %" PRId64 "\n", it->n_tokens); + it = slot.prompt.checkpoints.erase(it); + } else { + ++it; + } + } + } + auto & cur = slot.prompt.checkpoints.emplace_back(); cur.id_task = id_task; @@ -5252,7 +5268,7 @@ struct server_context_impl { common_speculative_get_draft_params(spec, slot.stream_slot_idx) = { // MAD-LAB: spec is now a raw ptr /* .drafting = */ true, /* .n_max = */ n_draft_max, - /* .n_past = */ slot.prompt.n_tokens(), + /* .pos0 = */ slot.prompt.tokens.pos_next(), /* .id_last = */ slot.sampled, /* .prompt = */ &slot.spec_prompt, /* .result = */ &slot.spec_draft, @@ -7983,7 +7999,7 @@ void server_routes::init_routes() { }; this->post_chat_completions_tok = [this](const server_http_req & req) { - return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_OAI_CHAT); + return handle_count_tokens(req, TASK_RESPONSE_TYPE_OAI_CHAT); }; this->post_control = [this](const server_http_req & req) { @@ -8042,7 +8058,7 @@ void server_routes::init_routes() { }; this->post_responses_tok_oai = [this](const server_http_req & req) { - return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_OAI_RESP); + return handle_count_tokens(req, TASK_RESPONSE_TYPE_OAI_RESP); }; this->post_transcriptions_oai = [this](const server_http_req & req) { @@ -8092,7 +8108,7 @@ void server_routes::init_routes() { }; this->post_anthropic_count_tokens = [this](const server_http_req & req) { - return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_ANTHROPIC); + return handle_count_tokens(req, TASK_RESPONSE_TYPE_ANTHROPIC); }; // same with handle_chat_completions, but without inference part @@ -8524,7 +8540,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons return res; } -std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const mtmd_helper_init_opt & init_opt, const server_http_req & req, task_response_type res_type) { +std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const server_http_req & req, task_response_type res_type) { auto res = create_response(); std::vector<raw_buffer> files; json body = json::parse(req.body); @@ -8558,13 +8574,13 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const l // TODO @ngxson : refactor this code block, move this to server-common and reuse it in other places size_t n_tokens; - if (mctx != nullptr) { + if (ctx_server.mctx != nullptr) { if (!prompt.is_string()) { throw std::runtime_error("for mtmd, input prompt must be a string."); } - n_tokens = process_mtmd_prompt(mctx, prompt.get<std::string>(), files, init_opt, true).size(); + n_tokens = process_mtmd_prompt(ctx_server.mctx, prompt.get<std::string>(), files, ctx_server.init_opt, true).size(); } else { - n_tokens = tokenize_mixed(vocab, prompt, true, true).size(); + n_tokens = tokenize_mixed(ctx_server.vocab, prompt, true, true).size(); } json response = {{"input_tokens", static_cast<int64_t>(n_tokens)}}; diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 0acbbffa9e10..7265ccad156d 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -169,7 +169,7 @@ struct server_routes { std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot); std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot); std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type); - std::unique_ptr<server_res_generator> handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const mtmd_helper_init_opt & init_opt, const server_http_req & req, task_response_type res_type); + std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type); // using unique_ptr to allow late initialization of const std::unique_ptr<const server_context_meta> meta; diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 2ec137aa0786..e46a1f6f1020 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -18,14 +18,37 @@ class server_http_context::Impl { public: - std::unique_ptr<httplib::Server> srv; + std::vector<std::unique_ptr<httplib::Server>> servers; + std::vector<std::string> hosts; + std::vector<std::thread> threads; // one thread per listener + std::unique_ptr<httplib::ThreadPool> pool; // single pool shared among all listeners + int n_threads_http = 0; +}; + +class server_http_task_queue : public httplib::TaskQueue { + httplib::ThreadPool & pool; +public: + explicit server_http_task_queue(httplib::ThreadPool & pool) : pool(pool) {} + bool enqueue(std::function<void()> fn) override { return pool.enqueue(std::move(fn)); } + // note: must call join() to drain the pool + void shutdown() override { /* no-op */ } }; server_http_context::server_http_context() : pimpl(std::make_unique<Impl>()) {} -server_http_context::~server_http_context() = default; +server_http_context::~server_http_context() { + // just in case any exit paths that forget to call join() + try { + stop(); + join(); + } catch (const std::exception & e) { + SRV_ERR("failed to stop HTTP server: %s\n", e.what()); + } catch (...) { + SRV_ERR("%s", "failed to stop HTTP server\n"); + } +} static void log_server_request(const httplib::Request & req, const httplib::Response & res) { // skip logging requests that are regularly sent, to avoid log spam @@ -90,7 +113,6 @@ bool server_http_context::init(const common_params & params) { path_prefix = params.api_prefix; port = params.port; - hostname = params.hostname; if (gcp.enabled) { SRV_TRC("Google Cloud Platform compat: health route = %s, predict route = %s, port = %d\n", gcp.path_health.c_str(), gcp.path_predict.c_str(), gcp.port); @@ -102,7 +124,39 @@ bool server_http_context::init(const common_params & params) { port = gcp.port; } - auto & srv = pimpl->srv; + pimpl->hosts = params.hostnames; + size_t n_tcp_hosts = 0; + for (const auto & host : pimpl->hosts) { + if (!string_ends_with(host, ".sock")) { + n_tcp_hosts++; + } + } + if (port == 0 && n_tcp_hosts > 1) { + SRV_ERR("%s", "--port 0 is not supported with multiple TCP addresses\n"); + return false; + } + for (size_t i = 0; i < pimpl->hosts.size(); ++i) { + pimpl->servers.emplace_back(); + if (!init_listener(params)) { + return false; + } + // with multiple TCP addresses, [::] must not also claim 0.0.0.0 + if (n_tcp_hosts > 1) { + pimpl->servers.back()->set_ipv6_v6only(true); + } + } + + pimpl->n_threads_http = params.n_threads_http; + if (pimpl->n_threads_http < 1) { + // +4 threads for monitoring, health and MCP. + pimpl->n_threads_http = std::max(params.n_parallel + 4, static_cast<int32_t>(std::thread::hardware_concurrency() - 1)); + } + SRV_TRC("using %d threads for HTTP server\n", pimpl->n_threads_http); + return true; +} + +bool server_http_context::init_listener(const common_params & params) { + auto & srv = pimpl->servers.back(); #ifdef CPPHTTPLIB_OPENSSL_SUPPORT if (!params.ssl_file_key.empty() && !params.ssl_file_cert.empty()) { @@ -306,18 +360,8 @@ bool server_http_context::init(const common_params & params) { return httplib::Server::HandlerResponse::Unhandled; }); - auto n_threads_http = params.n_threads_http; - if (n_threads_http < 1) { - // +4 threads for monitoring, health and some threads reserved for MCP and other tasks in the future - n_threads_http = std::max(params.n_parallel + 4, static_cast<int32_t>(std::thread::hardware_concurrency() - 1)); - } - SRV_TRC("using %d threads for HTTP server\n", n_threads_http); - srv->new_task_queue = [n_threads_http] { - // spawn n_threads_http fixed thread (always alive), while allow up to 1024 max possible additional threads - // when n_threads_http is used, server will create new "dynamic" threads that will be destroyed after processing each request - // ref: https://github.com/yhirose/cpp-httplib/pull/2368 - const auto max_threads = static_cast<size_t>(n_threads_http + 1024); - return new httplib::ThreadPool(n_threads_http, max_threads); + srv->new_task_queue = [this] { + return new server_http_task_queue(*pimpl->pool); }; // @@ -432,47 +476,76 @@ bool server_http_context::init(const common_params & params) { bool server_http_context::start() { // Bind and listen - const auto & srv = pimpl->srv; - auto was_bound = false; - auto is_sock = false; - if (string_ends_with(std::string(hostname), ".sock")) { - is_sock = true; - SRV_TRC("%s", "setting address family to AF_UNIX\n"); - srv->set_address_family(AF_UNIX); - // bind_to_port requires a second arg, any value other than 0 should - // simply get ignored - was_bound = srv->bind_to_port(hostname, 8080); - } else { - SRV_TRC("%s", "binding port with default address family\n"); - // bind HTTP listen port - if (port == 0) { - const auto bound_port = srv->bind_to_any_port(hostname); - was_bound = (bound_port >= 0); + listening_addresses.clear(); + for (size_t i = 0; i < pimpl->servers.size(); ++i) { + const auto & srv = pimpl->servers[i]; + const auto & host = pimpl->hosts[i]; + const bool is_sock = string_ends_with(host, ".sock"); + bool was_bound; + if (is_sock) { + SRV_TRC("%s", "setting address family to AF_UNIX\n"); + srv->set_address_family(AF_UNIX); + // AF_UNIX ignores the port, but bind_to_port requires a nonzero value. + was_bound = srv->bind_to_port(host, 8080); + } else if (port == 0) { + const auto bound_port = srv->bind_to_any_port(host); + was_bound = bound_port >= 0; if (was_bound) { port = bound_port; } } else { - was_bound = srv->bind_to_port(hostname, port); + was_bound = srv->bind_to_port(host, port); + } + if (!was_bound) { + SRV_ERR("couldn't bind HTTP server socket, hostname: %s, port: %d\n", host.c_str(), port); + stop(); + listening_addresses.clear(); + return false; } + listening_addresses.push_back(is_sock ? string_format("unix://%s", host.c_str()) + : string_format("%s://%s:%d", is_ssl ? "https" : "http", common_http_format_host(host).c_str(), port)); } - if (!was_bound) { - SRV_ERR("couldn't bind HTTP server socket, hostname: %s, port: %d\n", hostname.c_str(), port); - return false; + // n_threads_http fixed threads (always alive), plus up to 1024 dynamic threads destroyed after each request + // ref: https://github.com/yhirose/cpp-httplib/pull/2368 + pimpl->pool = std::make_unique<httplib::ThreadPool>(pimpl->n_threads_http, pimpl->n_threads_http + 1024); + for (size_t i = 0; i < pimpl->servers.size(); ++i) { + const auto & srv = pimpl->servers[i]; + pimpl->threads.emplace_back([srv = srv.get(), addr = listening_addresses[i]] { + if (!srv->listen_after_bind()) { + SRV_ERR("listener on %s stopped unexpectedly\n", addr.c_str()); + } + }); + srv->wait_until_ready(); + if (!srv->is_running()) { + SRV_ERR("couldn't start HTTP listener on %s\n", listening_addresses[i].c_str()); + stop(); + join(); + listening_addresses.clear(); + return false; + } } - - // run the HTTP server in a thread - thread = std::thread([this] { pimpl->srv->listen_after_bind(); }); - srv->wait_until_ready(); - - listening_address = is_sock ? string_format("unix://%s", hostname.c_str()) - : string_format("%s://%s:%d", is_ssl ? "https" : "http", common_http_format_host(hostname).c_str(), port); return true; } void server_http_context::stop() const { - if (pimpl->srv) { - pimpl->srv->stop(); + for (const auto & srv : pimpl->servers) { + if (srv) { + srv->stop(); + } + } +} + +void server_http_context::join() { + for (auto & thread : pimpl->threads) { + if (thread.joinable()) { + thread.join(); + } + } + // Queued requests still refer to their servers until the workers finish. + if (pimpl->pool) { + pimpl->pool->shutdown(); + pimpl->pool.reset(); } } @@ -584,7 +657,7 @@ static void process_handler_response(server_http_req_ptr && request, server_http void server_http_context::get(const std::string & path, const server_http_context::handler_t & handler) const { handlers.emplace(path, handler); - pimpl->srv->Get(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) { + auto callback = [handler](const httplib::Request & req, httplib::Response & res) { server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{ get_params(req), get_headers(req), @@ -596,12 +669,16 @@ void server_http_context::get(const std::string & path, const server_http_contex }); server_http_res_ptr response = handler(*request); process_handler_response(std::move(request), response, res); - }); + }; + const std::string full_path = path_prefix + path; + for (const auto & srv : pimpl->servers) { + srv->Get(full_path, callback); + } } void server_http_context::post(const std::string & path, const server_http_context::handler_t & handler) const { handlers.emplace(path, handler); - pimpl->srv->Post(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) { + auto callback = [handler](const httplib::Request & req, httplib::Response & res) { std::string body = req.body; std::map<std::string, uploaded_file> files; @@ -643,12 +720,16 @@ void server_http_context::post(const std::string & path, const server_http_conte }); server_http_res_ptr response = handler(*request); process_handler_response(std::move(request), response, res); - }); + }; + const std::string full_path = path_prefix + path; + for (const auto & srv : pimpl->servers) { + srv->Post(full_path, callback); + } } void server_http_context::del(const std::string & path, const server_http_context::handler_t & handler) const { handlers.emplace(path, handler); - pimpl->srv->Delete(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) { + auto callback = [handler](const httplib::Request & req, httplib::Response & res) { server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{ get_params(req), get_headers(req), @@ -660,7 +741,11 @@ void server_http_context::del(const std::string & path, const server_http_contex }); server_http_res_ptr response = handler(*request); process_handler_response(std::move(request), response, res); - }); + }; + const std::string full_path = path_prefix + path; + for (const auto & srv : pimpl->servers) { + srv->Delete(full_path, callback); + } } // diff --git a/tools/server/server-http.h b/tools/server/server-http.h index 032b08d0d210..4554b20f4aaf 100644 --- a/tools/server/server-http.h +++ b/tools/server/server-http.h @@ -68,7 +68,6 @@ struct server_http_context { class Impl; std::unique_ptr<Impl> pimpl; - std::thread thread; // server thread std::atomic<bool> is_ready = false; // note: the handler should never throw exceptions @@ -76,7 +75,6 @@ struct server_http_context { mutable std::unordered_map<std::string, handler_t> handlers; std::string path_prefix; - std::string hostname; int port = 8080; bool is_ssl = false; @@ -86,6 +84,7 @@ struct server_http_context { bool init(const common_params & params); bool start(); void stop() const; + void join(); void get(const std::string & path, const handler_t & handler) const; void post(const std::string & path, const handler_t & handler) const; @@ -96,5 +95,8 @@ struct server_http_context { void register_gcp_compat() const; // for debugging - std::string listening_address; + std::vector<std::string> listening_addresses; + +private: + bool init_listener(const common_params & params); }; diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 2500297bf0f5..d82b631d7315 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -8,6 +8,7 @@ #include "preset.h" #include "download.h" #include "fit.h" +#include "hf-cache.h" #include "http.h" #include "subproc.h" @@ -59,30 +60,215 @@ static constexpr const char * ROUTER_LOAD_TIMEOUT = "LLAMA_SERVER_ROUTER_LOA static constexpr int DEFAULT_ROUTER_LOAD_TIMEOUT_S = 900; +// note: SIGPIPE is ignored by the server +static void request_child_exit(server_subproc & proc) { + FILE * stdin_file = proc.sproc.stdin_file(); + if (stdin_file) { + fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); + fflush(stdin_file); + } +} + // address for child process, this is needed because router may run on 0.0.0.0 // ref: https://github.com/ggml-org/llama.cpp/issues/17862 #define CHILD_ADDR "127.0.0.1" -struct server_subproc { - common_subproc sproc; // not yet spawned while in DOWNLOADING state - std::atomic<bool> stopped{false}; // set to cancel a download or signal child process exit +// single-threaded, watching all child processes at once +struct server_monitor { + server_monitor(server_models & models) : models(models) { + th = std::thread([this]() { run(); }); + } + + ~server_monitor() { + push({ cmd_t::QUIT, {}, "", 0, false }); + th.join(); + } + + // thread-safe + void watch(const std::string & name, std::shared_ptr<server_subproc> proc, server_child_mode mode, int port) { + child_t c; + c.name = name; + c.proc = std::move(proc); + c.mode = mode; + c.port = port; + if (!c.proc->has_output()) { + SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str()); + c.eof = true; + } + push({ cmd_t::WATCH, std::move(c), "", 0, false }); + } + + // thread-safe + void stop(const std::string & name, int stop_timeout, bool send_exit) { + push({ cmd_t::STOP, {}, name, stop_timeout, send_exit }); + } + +private: + struct child_t { + std::string name; + std::shared_ptr<server_subproc> proc; + server_child_mode mode = SERVER_CHILD_MODE_NORMAL; + int port = 0; + std::string buf; // partial line + bool eof = false; // output closed, waiting for the process to be reaped + int64_t deadline = 0; // force-kill time in ms, 0 when no stop is pending + }; + + struct cmd_t { + enum { WATCH, STOP, QUIT } type; + child_t child; + std::string name; + int stop_timeout; + bool send_exit; + }; + + void push(cmd_t && cmd) { + { + std::lock_guard<std::mutex> lk(mu); + cmds.push_back(std::move(cmd)); + } + waiter.wake(); + } + + // returns true if the loop should exit + bool handle_commands() { + std::deque<cmd_t> batch; + { + std::lock_guard<std::mutex> lk(mu); + batch.swap(cmds); + } + for (auto & cmd : batch) { + switch (cmd.type) { + case cmd_t::WATCH: + children.push_back(std::move(cmd.child)); + break; + case cmd_t::STOP: + // the newest child with this name is the one the registry knows + for (auto it = children.rbegin(); it != children.rend(); ++it) { + if (it->name != cmd.name) { + continue; + } + if (cmd.send_exit && !it->eof) { + request_child_exit(*it->proc); + } + it->deadline = ggml_time_ms() + (int64_t) cmd.stop_timeout * 1000; + break; + } + break; + case cmd_t::QUIT: + return true; + } + } + return false; + } - bool is_alive() { - return sproc.alive(); + // read what the child wrote, forward complete lines + void read_output(child_t & c) { + char chunk[4096]; + while (!c.eof) { + int n = c.proc->read_output(chunk, sizeof(chunk)); + if (n < 0) { + c.eof = true; + break; + } + if (n == 0) { + break; + } + c.buf.append(chunk, (size_t) n); + size_t start = 0; + while (true) { + size_t nl = c.buf.find('\n', start); + if (nl == std::string::npos) { + break; + } + std::string line = c.buf.substr(start, nl + 1 - start); + start = nl + 1; + on_line(c, line); + } + c.buf.erase(0, start); + if (c.buf.size() > max_line) { + c.buf.clear(); // a child that never writes a newline must not grow this without bound + } + } + if (c.eof && !c.buf.empty()) { + on_line(c, c.buf); + c.buf.clear(); + } } - void request_exit() { - FILE * stdin_file = sproc.stdin_file(); - if (stdin_file) { - fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); - fflush(stdin_file); + void on_line(child_t & c, const std::string & line) { + if (string_starts_with(line, CMD_CHILD_TO_ROUTER_STATE)) { + LOG_DBG("[%5d] %s", c.port, line.c_str()); // prevent spamming the log + models.handle_child_state(c.name, line); + } else { + LOG("[%5d] %s", c.port, line.c_str()); // forward log } - stopped.store(true, std::memory_order_relaxed); } - void terminate() { - sproc.terminate(); + void run() { + while (true) { + if (handle_commands()) { + return; + } + + // wait for output, a wakeup, or the next deadline; + // a child whose output closed is polled for its exit every 50 ms + int64_t now = ggml_time_ms(); + int64_t timeout = -1; + for (const auto & c : children) { + if (c.eof) { + timeout = timeout < 0 ? 50 : std::min<int64_t>(timeout, 50); + } + if (c.deadline) { + int64_t d = std::max<int64_t>(0, c.deadline - now); + timeout = timeout < 0 ? d : std::min(timeout, d); + } + } + std::vector<server_subproc *> procs; + std::vector<child_t *> owners; + for (auto & c : children) { + if (!c.eof) { + procs.push_back(c.proc.get()); + owners.push_back(&c); + } + } + std::vector<bool> ready; + waiter.wait(procs, ready, timeout); + for (size_t i = 0; i < owners.size(); i++) { + if (ready[i]) { + read_output(*owners[i]); + } + } + + // deadlines and exits + now = ggml_time_ms(); + for (auto it = children.begin(); it != children.end();) { + if (it->deadline && now >= it->deadline && !it->proc->stopped.load(std::memory_order_acquire)) { + SRV_WRN("force-killing model instance name=%s after timeout\n", it->name.c_str()); + it->proc->terminate(); + it->deadline = 0; + } + if (it->eof && !it->proc->is_alive()) { + int exit_code = it->proc->join(); + it->proc->stopped.store(true, std::memory_order_release); + models.on_child_exit(it->name, it->proc, it->mode, exit_code); + SRV_INF("instance name=%s exited with status %d\n", it->name.c_str(), exit_code); + it = children.erase(it); + } else { + ++it; + } + } + } } + + static constexpr size_t max_line = 1024 * 1024; + + server_models & models; + std::mutex mu; + std::deque<cmd_t> cmds; + std::vector<child_t> children; // monitor thread only + server_subproc::waiter waiter; + std::thread th; }; struct server_lru_sched { @@ -95,26 +281,23 @@ struct server_lru_sched { } // returns "" if no model can be given up - std::string pick_victim(std::unique_lock<std::mutex> & lk, const std::string & exclude) { + std::string pick_victim(std::unique_lock<std::mutex> & lk) { check_lock(lk); std::string victim; int64_t victim_last_used = 0; for (const auto & m : models.mapping) { - if (m.first == exclude) { - continue; - } // a busy model is mid-request, one still coming up has no request to finish if (m.second.req_count != 0 || !m.second.meta.is_ready_or_sleep()) { continue; } - // FORK GUARDS (2026-08-10 upstream sync). Upstream's victim filter knows - // neither of these, and losing either is silent: - // - pinned is a HARD HOLD. A pinned resident owns its GPU until a human - // unpins it (the DSWS / weight-paging case). Evicting one pulls the - // card out from under work that asked to keep it. - // - a model already being unloaded must not be picked again, or a - // concurrent unload_lru() selects a second victim and over-evicts. - if (m.second.meta.placement.pinned || models.stopping_models.count(m.first)) { + // FORK GUARD: pinned is a HARD HOLD. A pinned resident owns its GPU until a + // human unpins it (the DSWS / weight-paging case). Evicting one pulls the card + // out from under work that asked to keep it. + if (m.second.meta.placement.pinned) { + continue; + } + // already on its way out, or a queued request wants it + if (models.stopping_models.count(m.first) || find(m.first)) { continue; } if (victim.empty() || m.second.meta.last_used < victim_last_used) { @@ -134,8 +317,8 @@ struct server_lru_sched { SRV_INF("request for name=%s joined the queue, %d waiting\n", model_id.c_str(), e->n_waiters); return; } - queue.push_back({ model_id, 1, false, false }); - SRV_INF("models_max reached, request for name=%s queued at position %zu\n", + queue.push_back({ model_id, 1, false }); + SRV_INF("request for name=%s queued at position %zu\n", model_id.c_str(), queue.size()); } @@ -169,85 +352,67 @@ struct server_lru_sched { return true; } - // ok means the model is up: drop the entry, the other waiters just watch its status now + // on failure the entry is back in line; on success it stays until its waiters leave, + // so the model coming up is never picked as a victim before they use it void claim_done(std::unique_lock<std::mutex> & lk, const std::string & model_id, bool ok) { check_lock(lk); + if (ok) { + return; + } for (auto it = queue.begin(); it != queue.end(); ++it) { if (it->model_id == model_id) { - if (ok) { - queue.erase(it); - } else { - it->loading = false; - } + it->loading = false; return; } } } - // a model is on its way out for this entry, so other requests do not also give up one - void mark_slot_pending(std::unique_lock<std::mutex> & lk, const std::string & model_id) { + // evict idle models while queued requests outnumber the slots that are free or being freed + // caller must hold models.mutex; never blocks, so it is safe from any thread + void tick(std::unique_lock<std::mutex> & lk) { check_lock(lk); - if (entry_t * e = find(model_id)) { - e->slot_pending = true; - } - } - - // model_id went idle: give up its slot if a queued request needs one - // thread-safe, caller must NOT hold models.mutex - void on_model_idle(const std::string & model_id) { - if (models.base_params.models_max <= 0) { - return; // no limit, nothing is ever queued + if (models.base_params.models_max <= 0 || queue.empty()) { + return; } - { - std::unique_lock<std::mutex> lk(models.mutex); - if (queue.empty()) { - return; - } - size_t promised = 0; - bool has_unserved = false; - for (const auto & e : queue) { - if (e.needs_slot()) { - has_unserved = true; - } else { - promised++; + int n_running = 0; + int n_stopping = 0; + for (const auto & m : models.mapping) { + if (m.second.meta.is_running()) { + n_running++; + if (models.stopping_models.count(m.first)) { + n_stopping++; } } - if (!has_unserved) { - return; - } - if ((int) count_running() - (int) promised < models.base_params.models_max) { - return; // a slot is already on its way - } - // never give up a model that a queued request wants - for (const auto & e : queue) { - if (e.model_id == model_id) { - return; - } + } + int n_needed = 0; + int n_claimed = 0; // claimed the slot, but load() has not spawned yet + for (const auto & e : queue) { + if (!e.loading) { + n_needed++; + continue; } - auto it = models.mapping.find(model_id); - if (it == models.mapping.end() || it->second.req_count != 0 || !it->second.meta.is_ready_or_sleep()) { - return; + auto it = models.mapping.find(e.model_id); + if (it != models.mapping.end() && !it->second.meta.is_running()) { + n_claimed++; } - for (auto & e : queue) { - if (!e.slot_pending) { - e.slot_pending = true; - break; - } + } + int n_free = models.base_params.models_max - n_running + n_stopping - n_claimed; + while (n_free < n_needed) { + std::string victim = pick_victim(lk); + if (victim.empty()) { + return; // all remaining models are busy, wait for a request to end } + SRV_INF("evicting idle LRU name=%s for a queued request\n", victim.c_str()); + models.request_stop(victim); + n_free++; } - SRV_INF("model name=%s went idle, giving up its slot to a queued request\n", model_id.c_str()); - models.unload(model_id); } private: struct entry_t { std::string model_id; - int n_waiters; // requests waiting for this model - bool slot_pending; // a model is already being evicted for this entry - bool loading; // one of the waiters is doing the load right now - - // a slot is already coming, or already taken by the load in flight - bool needs_slot() const { return !slot_pending && !loading; } + int n_waiters; // requests waiting for this model + bool loading; // one of the waiters is doing the load right now }; entry_t * find(const std::string & model_id) { @@ -326,6 +491,7 @@ static void unset_reserved_args(common_preset & preset, bool unset_model_args) { preset.unset_option("LLAMA_ARG_SSL_KEY_FILE"); preset.unset_option("LLAMA_ARG_SSL_CERT_FILE"); preset.unset_option("LLAMA_API_KEY"); + preset.unset_option("LLAMA_ARG_API_KEY_FILE"); preset.unset_option("LLAMA_ARG_MODELS_DIR"); preset.unset_option("LLAMA_ARG_MODELS_MAX"); preset.unset_option("LLAMA_ARG_MODELS_PRESET"); @@ -494,9 +660,14 @@ server_models::server_models( base_params(params), base_env(get_environment()), base_preset(ctx_preset.load_from_args(argc, argv)), - sched(std::make_unique<server_lru_sched>(*this)) { - // clean up base preset + sched(std::make_unique<server_lru_sched>(*this)), + monitor(std::make_unique<server_monitor>(*this)) { + // propagate base params to child unset_reserved_args(base_preset, true); + + // do not propagate these options, but allow preset to explicitly set them + base_preset.unset_option("LLAMA_ARG_LOG_FILE"); + // set binary path try { bin_path = get_server_exec_path().string(); @@ -608,6 +779,10 @@ void server_models::reload_models_preset_if_changed(server_model_meta & meta) { } } +void server_models::instance_t::request_exit() const { + request_child_exit(*subproc); +} + void server_models::add_model(server_model_meta && meta) { if (mapping.find(meta.name) != mapping.end()) { throw std::runtime_error(string_format("model '%s' appears multiple times", meta.name.c_str())); @@ -663,7 +838,6 @@ void server_models::add_model(server_model_meta && meta) { std::string name = meta.name; mapping[name] = instance_t{ /* subproc */ std::make_shared<server_subproc>(), - /* th */ std::thread(), /* meta */ std::move(meta) }; } @@ -1253,14 +1427,15 @@ void server_models::ensure_gpu_placement(const std::string & name, server_model_ reserve_gpu_placement_locked(name, meta.placement); for (const auto & victim : evict) { SRV_INF("router placement: evicting %s to make room for %s (exclusive: one model per GPU)\n", victim.c_str(), name.c_str()); - stopping_models.insert(victim); auto it = mapping.find(victim); - if (it != mapping.end() && it->second.meta.status == SERVER_MODEL_STATUS_LOADING) { + const bool loading = it != mapping.end() && it->second.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { it->second.subproc->terminate(); } + // marks the victim stopping and hands the stop to the monitor (upstream #28555) + request_stop(victim, !loading); } if (!evict.empty()) { - cv_stop.notify_all(); cv.wait(lk, [&]() { for (const auto & victim : evict) { auto it = mapping.find(victim); @@ -1364,14 +1539,15 @@ void server_models::ensure_gpu_placement(const std::string & name, server_model_ for (const auto & victim : evict) { SRV_INF("router placement evicting name=%s for model %s\n", victim.c_str(), name.c_str()); - stopping_models.insert(victim); auto it = mapping.find(victim); - if (it != mapping.end() && it->second.meta.status == SERVER_MODEL_STATUS_LOADING) { + const bool loading = it != mapping.end() && it->second.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { it->second.subproc->terminate(); } + // marks the victim stopping and hands the stop to the monitor (upstream #28555) + request_stop(victim, !loading); } if (!evict.empty()) { - cv_stop.notify_all(); cv.wait(lk, [&]() { for (const auto & victim : evict) { auto it = mapping.find(victim); @@ -1402,12 +1578,12 @@ void server_models::load_models() { std::optional<std::filesystem::file_time_type> models_preset_loaded_mtime; // 1. cached models common_presets cached_models = ctx_preset.load_from_cache(); - SRV_INF("Loaded %zu cached model presets\n", cached_models.size()); + SRV_TRC("Loaded %zu cached model presets from %s\n", cached_models.size(), hf_cache::get_cache_path().c_str()); // 2. local models from --models-dir common_presets local_models; if (!base_params.models_dir.empty()) { local_models = ctx_preset.load_from_models_dir(base_params.models_dir); - SRV_INF("Loaded %zu local model presets from %s\n", local_models.size(), base_params.models_dir.c_str()); + SRV_TRC("Loaded %zu local model presets from %s\n", local_models.size(), base_params.models_dir.c_str()); } // 3. custom-path models from presets common_preset global = {}; @@ -1422,7 +1598,7 @@ void server_models::load_models() { SRV_WRN("models preset '%s' changed while loading; it will be retried before the next child spawn\n", base_params.models_preset.c_str()); } - SRV_INF("Loaded %zu custom model presets from %s\n", custom_presets.size(), base_params.models_preset.c_str()); + SRV_TRC("Loaded %zu custom model presets from %s\n", custom_presets.size(), base_params.models_preset.c_str()); } load_gpu_config(global); @@ -1465,21 +1641,25 @@ void server_models::load_models() { std::set<std::string> hidden_models; { std::set<std::string> preset_paths; - for (const auto & [name, preset] : custom_presets) { - std::string val; - if (!preset.get_option(COMMON_ARG_PRESET_DEDUP_CACHE_MODELS, val) || !common_arg_utils::is_truthy(val)) { - continue; - } + auto add_hf_path = [&preset_paths](const common_preset & preset, const char * repo_key, const char * file_key) { std::string hf_repo; - if (!preset.get_option("LLAMA_ARG_HF_REPO", hf_repo) || hf_repo.empty()) { - continue; + if (!preset.get_option(repo_key, hf_repo) || hf_repo.empty()) { + return; } std::string hf_file; - preset.get_option("LLAMA_ARG_HF_FILE", hf_file); + preset.get_option(file_key, hf_file); std::string path = common_download_resolve_path(hf_repo, hf_file); if (!path.empty()) { preset_paths.insert(path); } + }; + for (const auto & [name, preset] : custom_presets) { + std::string val; + if (!preset.get_option(COMMON_ARG_PRESET_DEDUP_CACHE_MODELS, val) || !common_arg_utils::is_truthy(val)) { + continue; + } + add_hf_path(preset, "LLAMA_ARG_HF_REPO", "LLAMA_ARG_HF_FILE"); + add_hf_path(preset, "LLAMA_ARG_SPEC_DRAFT_HF_REPO", "LLAMA_ARG_SPEC_DRAFT_MODEL"); } if (!preset_paths.empty()) { for (const auto & [name, preset] : cached_models) { @@ -1496,8 +1676,6 @@ void server_models::load_models() { } // Helpers that read `mapping` - must be called while holding the lock. - std::unordered_set<std::string> custom_names; - for (const auto & [name, preset] : custom_presets) custom_names.insert(name); auto join_set = [](const std::set<std::string> & s) { std::string result; for (const auto & v : s) { @@ -1507,13 +1685,19 @@ void server_models::load_models() { return result; }; auto log_available_models = [&]() { - SRV_INF("Available models (%zu) (*: custom preset)\n", mapping.size()); - for (const auto & [name, inst] : mapping) { - bool has_custom = custom_names.find(name) != custom_names.end(); - std::string info; - if (!inst.meta.aliases.empty()) info += " (aliases: " + join_set(inst.meta.aliases) + ")"; - if (!inst.meta.tags.empty()) info += " [tags: " + join_set(inst.meta.tags) + "]"; - SRV_INF(" %c %s%s\n", has_custom ? '*' : ' ', name.c_str(), info.c_str()); + SRV_INF("Available models (%zu):\n", mapping.size()); + if (mapping.empty()) { + SRV_INF("%s", " no models found on the system (visit https://llama.app/models for suggestions)\n"); + } else { + for (const auto & [name, inst] : mapping) { + const std::string source = server_model_source_to_string(inst.meta.source); + + std::string info; + if (!inst.meta.aliases.empty()) info += " (aliases: " + join_set(inst.meta.aliases) + ")"; + if (!inst.meta.tags.empty()) info += " [tags: " + join_set(inst.meta.tags) + "]"; + + SRV_INF(" [%10s] %s%s\n", source.c_str(), name.c_str(), info.c_str()); + } } }; auto apply_stop_timeout = [&]() { @@ -1555,9 +1739,7 @@ void server_models::load_models() { }; // Phase 2: acquire the lock once for all mapping mutations. - // We temporarily release it only when calling functions that acquire it internally - // (unload, load) or when joining threads (the monitoring thread calls update_status - // which locks the mutex, so joining while holding it would deadlock). + // We temporarily release it only when calling functions that acquire it internally (unload) std::unique_lock<std::mutex> lk(mutex); if (models_preset_loaded_mtime.has_value()) { @@ -1651,49 +1833,15 @@ void server_models::load_models() { return true; }); - // collect all threads to join in one pass while the lock is held: - // - monitoring threads from just-unloaded models (to_unload) - // - threads of finished downloads (DOWNLOADED), they acquire the mutex on exit - // - threads of already-UNLOADED models that are being removed from source - std::vector<std::thread> threads_to_join; - for (const auto & name : to_unload) { - auto it = mapping.find(name); - if (it != mapping.end() && it->second.th.joinable()) { - threads_to_join.push_back(std::move(it->second.th)); - } - } - for (auto & [name, inst] : mapping) { - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { - continue; // downloading models are not from config sources, leave them alone - } - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) { - // joining this thread under the lock deadlocks: it locks the mutex on its way out - if (inst.th.joinable()) { - threads_to_join.push_back(std::move(inst.th)); - } - continue; - } - if (final_presets.find(name) == final_presets.end() && !inst.meta.is_running() && inst.th.joinable()) { - threads_to_join.push_back(std::move(inst.th)); - } - } - - // join outside the lock - monitoring thread calls update_status (needs lock) - lk.unlock(); - for (auto & th : threads_to_join) th.join(); - lk.lock(); - // erase models no longer in any source for (auto it = mapping.begin(); it != mapping.end(); ) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { ++it; // download thread is still busy, skip } else if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) { - // download finished, thread is joined above, safe to erase - GGML_ASSERT(!it->second.th.joinable()); + // download finished, safe to erase it = mapping.erase(it); } else if (final_presets.find(it->first) == final_presets.end()) { SRV_INF("(reload) removing model name=%s (no longer in source)\n", it->first.c_str()); - GGML_ASSERT(!it->second.th.joinable()); // must have been joined above it = mapping.erase(it); } else { ++it; @@ -1872,25 +2020,26 @@ void server_models::unload_lru() { // remove one of the servers if we passed the models_max (least recently used - LRU) // 2026-08-10 upstream sync: eviction moved into upstream's server_lru_sched. // The fork's pinned / stopping_models guards that used to live in this loop are - // now enforced inside server_lru_sched::pick_victim -- see the FORK GUARDS note - // there. Do not reintroduce a second victim-selection path here. + // now enforced inside server_lru_sched::pick_victim (the FORK GUARD note plus + // upstream's stopping/queued filter). Do not reintroduce a second victim-selection + // path here. std::string lru_model_name; { std::unique_lock<std::mutex> lk(mutex); if (sched->has_capacity(lk)) { return; } - lru_model_name = sched->pick_victim(lk, ""); + lru_model_name = sched->pick_victim(lk); if (lru_model_name.empty()) { return; } - // Mark the victim as stopping under the SAME lock that selected it, so a - // concurrent unload_lru() excludes it in pick_victim and cannot evict a - // second model. unload() below re-inserts idempotently and triggers the stop. - stopping_models.insert(lru_model_name); + // Stop the victim under the SAME lock that selected it: request_stop() marks it + // stopping, so a concurrent unload_lru() excludes it in pick_victim and cannot + // evict a second model. pick_victim only returns ready/sleeping models, so a + // graceful exit request is always right here. + SRV_INF("models_max limit reached, removing LRU name=%s\n", lru_model_name.c_str()); + request_stop(lru_model_name, true); } - SRV_INF("models_max limit reached, removing LRU name=%s\n", lru_model_name.c_str()); - unload(lru_model_name); // wait for unload to complete (find-based: safe if the entry was erased mid-wait, // unlike the previous mapping[name] which default-constructed a stray entry) wait(lru_model_name, [](const server_model_meta & meta) { @@ -1972,7 +2121,6 @@ void server_models::load(const std::string & name, const load_options & opts) { stopping_models.erase(name); marked_loading = false; cv.notify_all(); - cv_stop.notify_all(); }; auto rollback_load_attempt = [&](void *) { if (!lk.owns_lock()) { @@ -1995,7 +2143,8 @@ void server_models::load(const std::string & name, const load_options & opts) { // concurrent load claimed the slot unload_lru() just freed) we evict again // and wait for a slot rather than failing the request with a 500 — the // request effectively queues behind the eviction it triggers. - if (!gpu_placement_enabled && base_params.models_max > 0) { + // Download workers do not use models_max slots. + if (!gpu_placement_enabled && opts.mode == SERVER_CHILD_MODE_NORMAL && base_params.models_max > 0) { const int64_t capacity_deadline = ggml_time_ms() + 30000; // 30s cap, then surface a retriable error auto count_running = [this]() { size_t n = 0; @@ -2098,117 +2247,12 @@ void server_models::load(const std::string & name, const load_options & opts) { load_attempt_guard.release(); } - // start a thread to manage the child process - // captured variables are guaranteed to be destroyed only after the thread is joined - inst.th = std::thread([ - this, name, - child_proc = inst.subproc, - port = inst.meta.port, - stop_timeout = inst.meta.stop_timeout, - child_mode = opts.mode - ]() { - FILE * stdin_file = child_proc->sproc.stdin_file(); - FILE * stdout_file = child_proc->sproc.stdout_file(); // combined stdout/stderr - - std::thread log_thread([&]() { - // read stdout/stderr and forward to main server log - // also handle status report from child process - std::vector<char> vec_buf(128 * 1024); // large buffer for storing info - char * buffer = vec_buf.data(); - if (stdout_file) { - while (fgets(buffer, vec_buf.size(), stdout_file) != nullptr) { - std::string str(buffer); - if (string_starts_with(buffer, CMD_CHILD_TO_ROUTER_STATE)) { - LOG_DBG("[%5d] %s", port, buffer); // prevent spamming the log - this->handle_child_state(name, str); - } else { - // forward log - LOG("[%5d] %s", port, buffer); - } - } - } else { - SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str()); - } - }); - - std::thread stopping_thread([&]() { - // thread to monitor explicit stop requests; child crash is signalled via child_proc->stopped - auto is_stopping = [this, &name]() { - return this->stopping_models.find(name) != this->stopping_models.end(); - }; - { - std::unique_lock<std::mutex> lk(this->mutex); - this->cv_stop.wait(lk, [&]() { - return is_stopping() || child_proc->stopped.load(std::memory_order_acquire); - }); - } - // child crashed or finished on its own, skip graceful shutdown sequence - if (child_proc->stopped.load(std::memory_order_acquire)) { - return; - } - SRV_INF("stopping model instance name=%s\n", name.c_str()); - fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); - fflush(stdin_file); - int64_t start_time = ggml_time_ms(); - while (true) { - std::unique_lock<std::mutex> lk(this->mutex); - if (!is_stopping() || child_proc->stopped.load(std::memory_order_acquire)) { - return; - } - int64_t elapsed = ggml_time_ms() - start_time; - if (elapsed >= stop_timeout * 1000) { - lk.unlock(); - SRV_WRN("force-killing model instance name=%s after %d seconds timeout\n", name.c_str(), stop_timeout); - child_proc->terminate(); - return; - } - this->cv_stop.wait_for(lk, std::chrono::seconds(1), [&]() { - return !is_stopping() || child_proc->stopped.load(std::memory_order_acquire); - }); - } - }); - - // we reach here when the child process exits (stdout EOF) - // note: we cannot join() prior to this point because it will close stdin_file - if (log_thread.joinable()) { - log_thread.join(); - } - - child_proc->stopped.store(true, std::memory_order_release); - { - std::lock_guard<std::mutex> lk(this->mutex); - stopping_models.erase(name); - cv_stop.notify_all(); - } - if (stopping_thread.joinable()) { - stopping_thread.join(); - } - - // get the exit code - int exit_code = child_proc->sproc.join(); - - // update status and exit code - if (child_mode == SERVER_CHILD_MODE_DOWNLOAD) { - // instance will be cleaned up on next load_models() call - } else { - this->update_status(name, { - SERVER_MODEL_STATUS_UNLOADED, - exit_code - }); - } - SRV_INF("instance name=%s exited with status %d\n", name.c_str(), exit_code); - }); - - // clean up old process/thread if exists + // old process should have exited already, but just in case, we clean it up here { - auto & old_instance = mapping[name]; - // old process should have exited already, but just in case, we clean it up here - if (old_instance.subproc && old_instance.subproc->is_alive()) { + auto it = mapping.find(name); + if (it != mapping.end() && it->second.subproc && it->second.subproc->is_alive()) { SRV_WRN("old process for model name=%s is still alive, this is unexpected\n", name.c_str()); - old_instance.subproc->terminate(); // force kill - } - if (old_instance.th.joinable()) { - old_instance.th.join(); + it->second.subproc->terminate(); // force kill } } @@ -2216,31 +2260,65 @@ void server_models::load(const std::string & name, const load_options & opts) { {"status", server_model_status_to_string(inst.meta.status)}, }); + auto proc = inst.subproc; + int port = inst.meta.port; mapping[name] = std::move(inst); + monitor->watch(name, proc, opts.mode, port); cv.notify_all(); } +void server_models::request_stop(const std::string & name, bool send_exit) { + auto it = mapping.find(name); + if (it == mapping.end() || stopping_models.count(name)) { + return; + } + stopping_models.insert(name); + monitor->stop(name, it->second.meta.stop_timeout, send_exit); +} + +void server_models::on_child_exit(const std::string & name, const std::shared_ptr<server_subproc> & proc, server_child_mode mode, int exit_code) { + { + std::lock_guard<std::mutex> lk(mutex); + auto it = mapping.find(name); + if (it == mapping.end() || it->second.subproc != proc) { + stopping_models.erase(name); + return; // entry erased, or a newer instance took the name + } + } + if (mode == SERVER_CHILD_MODE_DOWNLOAD) { + // instance will be cleaned up on next load_models() call + std::lock_guard<std::mutex> lk(mutex); + stopping_models.erase(name); + cv.notify_all(); + } else { + update_status(name, { + SERVER_MODEL_STATUS_UNLOADED, + exit_code + }); + } +} + void server_models::unload(const std::string & name) { std::unique_lock<std::mutex> lk(mutex); auto it = mapping.find(name); if (it != mapping.end()) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { SRV_INF("cancelling download for model name=%s\n", name.c_str()); - it->second.subproc->request_exit(); + it->second.request_exit(); // for convenience, we wait the status change here wait(lk, name, [](const server_model_meta & new_meta) { return new_meta.status != SERVER_MODEL_STATUS_DOWNLOADING; }); } else if (it->second.meta.is_running()) { SRV_INF("stopping model instance name=%s\n", name.c_str()); - stopping_models.insert(name); - if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) { + bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { // special case: if model is in loading state, unloading means force-killing it SRV_WRN("model name=%s is still loading, force-killing\n", name.c_str()); it->second.subproc->terminate(); } - cv_stop.notify_all(); - // status change will be handled by the managing thread + request_stop(name, !loading); + // status change will be handled by the monitor } else { SRV_WRN("model instance name=%s is not running\n", name.c_str()); } @@ -2248,28 +2326,29 @@ void server_models::unload(const std::string & name) { } void server_models::unload_all() { - std::vector<std::thread> to_join; - { - std::lock_guard<std::mutex> lk(mutex); - for (auto & [name, inst] : mapping) { - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { - SRV_INF("cancelling download for model name=%s\n", name.c_str()); - inst.subproc->stopped.store(true, std::memory_order_relaxed); - } else if (inst.meta.is_running()) { - SRV_INF("stopping model instance name=%s\n", name.c_str()); - stopping_models.insert(name); - cv_stop.notify_all(); - // status change will be handled by the managing thread + std::unique_lock<std::mutex> lk(mutex); + for (auto & [name, inst] : mapping) { + if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + SRV_INF("cancelling download for model name=%s\n", name.c_str()); + inst.request_exit(); + } else if (inst.meta.is_running()) { + SRV_INF("stopping model instance name=%s\n", name.c_str()); + bool loading = inst.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { + inst.subproc->terminate(); } - // moving the thread to join list to avoid deadlock - to_join.push_back(std::move(inst.th)); + request_stop(name, !loading); } } - for (auto & th : to_join) { - if (th.joinable()) { - th.join(); + // wait for every child to exit, the monitor force-kills the ones that ignore the exit command + cv.wait(lk, [this]() { + for (const auto & [name, inst] : mapping) { + if (inst.meta.is_running() || inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + return false; + } } - } + return true; + }); } void server_models::update_status(const std::string & name, const update_status_args & args) { @@ -2282,6 +2361,9 @@ void server_models::update_status(const std::string & name, const update_status_ } meta.status = args.status; meta.exit_code = args.exit_code; + if (args.status == SERVER_MODEL_STATUS_UNLOADED) { + stopping_models.erase(name); + } if (!args.loaded_info.is_null()) { meta.loaded_info = args.loaded_info; } @@ -2291,6 +2373,8 @@ void server_models::update_status(const std::string & name, const update_status_ if (args.status == SERVER_MODEL_STATUS_LOADED && !meta.placement.need_bytes_per_dev.empty()) { reconcile_gpu_reservation_locked(name); } + // a model that comes up idle or goes down changes the slot count for queued requests + sched->tick(lk); } // broadcast status change to SSE { @@ -2360,18 +2444,18 @@ bool server_models::remove(const std::string & name) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { // cancel in-flight download SRV_INF("cancelling download for model name=%s\n", name.c_str()); - it->second.subproc->request_exit(); + it->second.request_exit(); } else if (it->second.meta.is_running()) { // stop running instance SRV_INF("stopping model instance name=%s\n", name.c_str()); - stopping_models.insert(name); - if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) { + bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { it->second.subproc->terminate(); } - cv_stop.notify_all(); + request_stop(name, !loading); } - // wait until the monitoring thread finishes + // wait until the child is gone wait(lk, name, [](const server_model_meta & meta) { return meta.status == SERVER_MODEL_STATUS_UNLOADED || meta.status == SERVER_MODEL_STATUS_DOWNLOADED; @@ -2380,8 +2464,7 @@ bool server_models::remove(const std::string & name) { // re-find after wait - load_models() may have erased the entry during the wait it = mapping.find(name); if (it == mapping.end()) { - // load_models() already joined the thread and erased the entry; - // we just need to clean up the cached files on disk + // load_models() already erased the entry; we just need to clean up the cached files on disk lk.unlock(); bool ok = common_download_remove(name); SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial"); @@ -2389,11 +2472,6 @@ bool server_models::remove(const std::string & name) { return true; } - // join before erasing - thread no longer acquires this mutex - if (it->second.th.joinable()) { - it->second.th.join(); - } - // remove from disk (best-effort: cancelled downloads may have no cached files) bool ok = common_download_remove(name); mapping.erase(name); @@ -2428,44 +2506,30 @@ bool server_models::ensure_model_ready(const std::string & name, const std::func if (!meta.has_value()) { throw std::runtime_error("model name=" + name + " is not found"); } - if (meta->is_ready()) { + bool stopping; + { + std::lock_guard<std::mutex> lk(mutex); + stopping = stopping_models.count(name) > 0; + } + if (!stopping && meta->is_ready()) { return false; // ready for taking requests } - if (meta->status == SERVER_MODEL_STATUS_SLEEPING) { + if (!stopping && meta->status == SERVER_MODEL_STATUS_SLEEPING) { return false; // child is sleeping but still running; new request will wake it up } bool queued = false; bool did_load = false; - std::string victim; { std::unique_lock<std::mutex> lk(mutex); auto it = mapping.find(name); if (it != mapping.end() && it->second.meta.status == SERVER_MODEL_STATUS_UNLOADED) { - bool has_capacity = sched->has_capacity(lk); - if (has_capacity && sched->queue_empty(lk)) { - lk.unlock(); - SRV_INF("model name=%s is not loaded, loading...\n", name.c_str()); - load(name); - did_load = true; - } else { - // also queue when a slot looks free but others wait already, else they starve - sched->join(lk, name); - queued = true; - if (!has_capacity) { - // an idle model may sit here right now, do not wait for a request to end - victim = sched->pick_victim(lk, name); - if (!victim.empty()) { - sched->mark_slot_pending(lk, name); - } - } - } + // the queue entry protects the model from eviction until its waiters leave + sched->join(lk, name); + sched->tick(lk); + queued = true; } } - if (!victim.empty()) { - SRV_INF("evicting idle LRU name=%s to make room for name=%s\n", victim.c_str(), name.c_str()); - unload(victim); - } // while queued, this is also where the load happens: the head of the queue does it SRV_INF("waiting until model name=%s is fully loaded...\n", name.c_str()); @@ -2484,6 +2548,19 @@ bool server_models::ensure_model_ready(const std::string & name, const std::func if (it == mapping.end()) { break; // removed by another code path, nothing to wait for } + if (stopping_models.count(name)) { + // a stopping instance takes no new request, the next instance serves it + if (!queued) { + sched->join(lk, name); + sched->tick(lk); + queued = true; + } + if (should_stop && should_stop()) { + throw std::runtime_error("request cancelled while waiting for model name=" + name); + } + cv.wait_for(lk, std::chrono::milliseconds(200)); + continue; + } const server_model_status status = it->second.meta.status; if (status == SERVER_MODEL_STATUS_LOADED || status == SERVER_MODEL_STATUS_SLEEPING) { @@ -2527,9 +2604,7 @@ bool server_models::ensure_model_ready(const std::string & name, const std::func } lk.lock(); sched->claim_done(lk, name, ok); - if (ok) { - queued = false; // entry is gone, the other waiters watch the status now - } + sched->tick(lk); continue; } @@ -2537,6 +2612,7 @@ bool server_models::ensure_model_ready(const std::string & name, const std::func } } catch (...) { leave_queue(); + sched->tick(lk); // a slot freed for this waiter goes to the next one throw; } leave_queue(); @@ -2590,30 +2666,25 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co base_params.timeout_write ); - // Chain onto the proxy's own cleanup (it sets one in its ctor -- do NOT clobber it; - // upstream's version assigns over it). When the proxy dies the request is done, so - // drop the in-flight count and re-stamp last_used: the idle clock should start when - // the response ENDS, not when it began. + // Chain onto any cleanup the proxy already carries (do NOT clobber it). When the proxy + // dies the request is done, so drop the in-flight count and re-stamp last_used: the + // idle clock should start when the response ENDS, not when it began. auto prev_cleanup = proxy->cleanup; proxy->cleanup = [this, name, prev_cleanup]() { if (prev_cleanup) { prev_cleanup(); } - bool went_idle = false; - { - std::unique_lock<std::mutex> lk(mutex); - auto it = mapping.find(name); - if (it != mapping.end()) { - if (it->second.req_count > 0) { - it->second.req_count--; - went_idle = it->second.req_count == 0; + std::unique_lock<std::mutex> lk(mutex); + auto it = mapping.find(name); + if (it != mapping.end()) { + it->second.meta.last_used = ggml_time_ms(); + if (it->second.req_count > 0) { + it->second.req_count--; + if (it->second.req_count == 0) { + sched->tick(lk); } - it->second.meta.last_used = ggml_time_ms(); } } - if (went_idle) { - sched->on_model_idle(name); - } }; @@ -2736,7 +2807,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri std::lock_guard<std::mutex> lk(mutex); auto it = mapping.find(name); if (it != mapping.end()) { - return it->second.subproc->request_exit(); + return it->second.request_exit(); } }; if (result == "download_finished") { @@ -2953,7 +3024,10 @@ void server_child::notify_to_router(const std::string & state, const json & payl std::lock_guard<std::mutex> lk(mtx_stdout); common_log_pause(common_log_main()); fflush(stdout); - fprintf(stdout, "%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str()); + // the router matches the command on a line prefix, so the leading newline + // closes whatever the logger left open on the shared pipe, down to the + // trailing color reset that carries no newline of its own + fprintf(stdout, "\n%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str()); fflush(stdout); common_log_resume(common_log_main()); } diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 41b51d46ea46..a924c8f4d0c5 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -12,6 +12,7 @@ #include <mutex> #include <condition_variable> #include <filesystem> +#include <thread> #include <functional> #include <memory> #include <optional> @@ -149,17 +150,17 @@ struct server_model_meta { }; struct server_models_routes; -struct server_subproc; // defined in server-models.cpp struct server_lru_sched; // defined in server-models.cpp +struct server_monitor; // defined in server-models.cpp struct server_models { friend struct server_models_routes; friend struct server_lru_sched; + friend struct server_monitor; private: struct instance_t { - std::shared_ptr<server_subproc> subproc; // shared between main thread and monitoring thread - std::thread th; + std::shared_ptr<server_subproc> subproc; // shared with the monitor thread server_model_meta meta; // Requests currently being proxied to this model. The idle sweeper refuses to // unload a model with any in flight -- `meta.last_used` is stamped when a request @@ -169,6 +170,9 @@ struct server_models { // same counter as `req_count` and its scheduler reads that name, so the two are // unified here rather than kept side by side. int req_count = 0; // number of active proxy requests + + // ask the child to exit (it handles the command on its stdin, see server_child::setup) + void request_exit() const; }; std::mutex mutex; @@ -192,8 +196,7 @@ struct server_models { std::atomic<bool> idle_stop{false}; void idle_sweeper_loop(); - // for stopping models - std::condition_variable cv_stop; + // models asked to stop, still counted as running until the monitor records their exit std::set<std::string> stopping_models; // set to true while load_models() is executing a reload; load() will wait until clear @@ -310,6 +313,13 @@ struct server_models { // not thread-safe, caller must hold mutex void add_model(server_model_meta && meta); + // ask the monitor to stop a running instance; send_exit is false for a child that was already force-killed + // not thread-safe, caller must hold mutex + void request_stop(const std::string & name, bool send_exit = true); + + // called by the monitor once a child exited and was reaped + void on_child_exit(const std::string & name, const std::shared_ptr<server_subproc> & proc, server_child_mode mode, int exit_code); + // notify SSE clients void notify_sse(const std::string & event, const std::string & model_id, const json & data = nullptr); @@ -387,12 +397,16 @@ struct server_models { // handle message sent from server_child::notify_to_router() // raw input must starts with CMD_CHILD_TO_ROUTER_STATE, followed by a JSON string - // this function is not thread-safe, must be called from instance's monitoring thread + // called from the monitor thread // payload per state: // state = loading -> payload = {} (TODO: add progress info) // state = ready -> payload = model_info (json), or {} if wakeup from sleeping // state = sleeping -> payload = {} void handle_child_state(const std::string & name, const std::string & raw_input); + +private: + // one thread watching every child; keep last, the destructor joins the thread + std::unique_ptr<server_monitor> monitor; }; struct server_child { diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index 64b9251295ce..27ecafb7a595 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -257,6 +257,10 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params & if (data.contains("json_schema") && !data.contains("grammar")) { try { auto schema = json_value(data, "json_schema", json::object()); + if (schema.is_object() && schema.empty()) { + // an empty schema means any object + schema["type"] = "object"; + } SRV_DBG("JSON schema: %s\n", schema.dump(2).c_str()); std::string grammar_str = json_schema_to_grammar(schema); SRV_DBG("Converted grammar: %s\n", grammar_str.c_str()); diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 6584305a7c2c..062a08c57482 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -106,6 +106,8 @@ int llama_server(int argc, char ** argv) { // touch it. lifecycle is symmetric, stop_gc() runs in clean_up() before backend free server_stream_session_manager_start(); + SRV_INF("%s", "initializing ...\n"); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) { return 1; } @@ -124,7 +126,9 @@ int llama_server(int argc, char ** argv) { return rc; } - return llama_server(params, argc, argv); + const int result = llama_server(params, argc, argv); + common_log_flush(common_log_main()); + return result; } int llama_server(common_params & params, int argc, char ** argv) { @@ -233,12 +237,6 @@ int llama_server(common_params & params, int argc, char ** argv) { // struct that contains llama context and inference server_context ctx_server; - server_http_context ctx_http; - if (!ctx_http.init(params)) { - SRV_ERR("%s", "failed to initialize HTTP server\n"); - return 1; - } - // // Router // @@ -249,6 +247,13 @@ int llama_server(common_params & params, int argc, char ** argv) { server_tools tools; std::optional<server_models_routes> models_routes{}; + + server_http_context ctx_http; + if (!ctx_http.init(params)) { + SRV_ERR("%s", "failed to initialize HTTP server\n"); + return 1; + } + if (is_router_server) { // setup server instances manager try { @@ -374,11 +379,7 @@ int llama_server(common_params & params, int argc, char ** argv) { }; if (params.cors_origins == "*" && params.api_keys.empty()) { - SRV_WRN("%s", "-----------------\n"); - SRV_WRN("%s", "CORS is set to allow all origins ('*') and no API key is set\n"); - SRV_WRN("%s", "this can be a security risk (cross-origin attacks)\n"); - SRV_WRN("%s", "more info: https://github.com/ggml-org/llama.cpp/pull/25655\n"); - SRV_WRN("%s", "-----------------\n"); + SRV_WRN("%s", "security: no API key is set and CORS allows all origins (see https://github.com/ggml-org/llama.cpp/pull/25655)\n"); } // CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP) @@ -426,14 +427,13 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.post("/tools", ex_wrapper(res_403)); } - if (warn_names.size() > 0) { - SRV_WRN("%s", "-----------------\n"); - SRV_WRN("%s", "the following feature(s) are enabled:\n"); + if (!warn_names.empty()) { + std::string features; for (const auto & name : warn_names) { - SRV_WRN(" %s\n", name.c_str()); + if (!features.empty()) features += ", "; + features += name; } - SRV_WRN("%s", "do not expose the server to untrusted environments\n"); - SRV_WRN("%s", "-----------------\n"); + SRV_WRN("security: %s enabled - do not expose to untrusted environments\n", features.c_str()); } // @@ -503,9 +503,7 @@ int llama_server(common_params & params, int argc, char ** argv) { } catch (const std::exception & e) { SRV_ERR("failed to load models on startup: %s\n", e.what()); ctx_http.stop(); - if (ctx_http.thread.joinable()) { - ctx_http.thread.join(); - } + ctx_http.join(); clean_up(); return 1; } @@ -538,9 +536,7 @@ int llama_server(common_params & params, int argc, char ** argv) { if (!ctx_server.load_model(params)) { clean_up(); - if (ctx_http.thread.joinable()) { - ctx_http.thread.join(); - } + ctx_http.join(); SRV_ERR("%s", "exiting due to model loading error\n"); return 1; } @@ -574,13 +570,16 @@ int llama_server(common_params & params, int argc, char ** argv) { #endif } - SRV_INF("listening on %s\n", ctx_http.listening_address.c_str()); + bool uses_default_port = false; + for (const auto & address : ctx_http.listening_addresses) { + SRV_INF("listening on %s\n", address.c_str()); + uses_default_port |= string_ends_with(address, ":8080"); + } // TODO: remove this in the future // check the string to also handle the .sock case - if (string_ends_with(ctx_http.listening_address, ":8080")) { - SRV_WRN("%s", "NOTICE: server default port will be changed to :9931 in a future release\n"); - SRV_WRN("%s", " ref: https://github.com/ggml-org/llama.cpp/pull/26508\n"); + if (uses_default_port) { + SRV_WRN("%s", "notice: server default port will be changed to :9931 in a future release (ref: https://github.com/ggml-org/llama.cpp/pull/26508)\n"); } if (is_router_server) { @@ -589,9 +588,7 @@ int llama_server(common_params & params, int argc, char ** argv) { SRV_WRN("%s", " please only use presets that you can trust! Unknown presets may be unsafe\n"); } - if (ctx_http.thread.joinable()) { - ctx_http.thread.join(); // keep the main thread alive - } + ctx_http.join(); // keep the main thread alive // when the HTTP server stops, clean up and exit clean_up(); @@ -607,9 +604,7 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_server.start_loop(); clean_up(); - if (ctx_http.thread.joinable()) { - ctx_http.thread.join(); - } + ctx_http.join(); if (monitor_thread.joinable()) { monitor_thread.join(); } diff --git a/tools/server/tests/requirements.txt b/tools/server/tests/requirements.txt index 6c256f67d838..5e6dff298f4e 100644 --- a/tools/server/tests/requirements.txt +++ b/tools/server/tests/requirements.txt @@ -2,7 +2,7 @@ aiohttp~=3.9.3 pytest~=8.3.3 pytest-xdist~=3.6 filelock~=3.16 -numpy~=1.26.4 +numpy~=2.2.6 openai~=2.14.0 prometheus-client~=0.20.0 requests~=2.32.3 diff --git a/tools/server/tests/unit/test_basic.py b/tools/server/tests/unit/test_basic.py index 285726abf406..b9e9f84f6383 100644 --- a/tools/server/tests/unit/test_basic.py +++ b/tools/server/tests/unit/test_basic.py @@ -1,5 +1,6 @@ import pytest import requests +import socket from utils import * server = ServerPreset.tinyllama2() @@ -18,6 +19,37 @@ def test_server_start_simple(): assert res.status_code == 200 +def test_server_multiple_addresses(monkeypatch): + # The CLI value replaces the environment value, including an unavailable address. + monkeypatch.setenv("LLAMA_ARG_HOST", "192.0.2.1") + try: + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as probe: + probe.bind(("::1", 0)) + except OSError: + pytest.skip("IPv6 loopback is unavailable") # ty: ignore[too-many-positional-arguments] + + server.server_host = "127.0.0.1,::1" + server.api_key = "test-multiple-addresses" + server.start() + + def check_address(host): + res = server.make_request("GET", "/health", host=host) + assert res.status_code == 200 + res = server.make_request("POST", "/v1/completions", data={}, host=host) + assert res.status_code == 401 + events = list(server.make_stream_request("POST", "/v1/completions", data={ + "prompt": "Once upon a time", + "max_tokens": 8, + "stream": True, + }, headers={"Authorization": f"Bearer {server.api_key}"}, host=host)) + assert len(events) > 1 + return True + + # parallel_function_calls swallows exceptions, a failed check leaves None in the results + results = parallel_function_calls([(check_address, (host,)) for host in ["127.0.0.1", "[::1]"]]) + assert all(results) + + def test_server_props(): global server server.start() diff --git a/tools/server/tests/unit/test_completion.py b/tools/server/tests/unit/test_completion.py index 9375e0110e53..01732eb16313 100644 --- a/tools/server/tests/unit/test_completion.py +++ b/tools/server/tests/unit/test_completion.py @@ -394,7 +394,12 @@ def test_completion_unified(n_ctx, n_slots, n_predict_vals, expected_success): results = parallel_function_calls(tasks) for res, n_predict, expect_ok in zip(results, n_predict_vals, expected_success): if expect_ok: - assert res.status_code == 200 + # the pool is aborted as a whole, so a request that fits on its own + # is still dropped when the slots overlap, and it says so explicitly + assert res.status_code == 200 or ( + res.status_code == 500 + and "context size has been exceeded" in res.body["error"]["message"].lower() + ) # note: https://github.com/ggml-org/llama.cpp/pull/18700#issuecomment-3728695581 if res.status_code == 200: diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.py index 96eb87978f58..5355ab7367ed 100644 --- a/tools/server/tests/unit/test_router.py +++ b/tools/server/tests/unit/test_router.py @@ -297,6 +297,26 @@ def test_router_queue_is_fifo(): assert first.done_at < second.done_at, "queue was not served in arrival order" +def test_router_queue_two_waiters_share_one_eviction(): + """two requests that both find the same idle model must both be served in the end""" + global server + server.models_max = 1 + server.start() + + _load_model_and_wait(MODEL_A, timeout=120) + + # both arrive while MODEL_A is idle, so both want its slot; only one eviction can happen + first = _Bg(lambda: _tokenize(MODEL_B)).start() + second = _Bg(lambda: _tokenize(MODEL_C)).start() + + first.join(90) + second.join(90) + + first.assert_ok("first queued request") + second.assert_ok("second queued request") + assert _get_model_status(MODEL_A) == "unloaded" + + def test_router_no_models_autoload(): global server server.no_models_autoload = True @@ -413,12 +433,14 @@ def test_router_dedup_cache_models(): global server preset_path = os.path.join(TMP_DIR, "test_dedup.ini") - cache_id = "ggml-org/test-model-stories260K:F32" + main_cache_id = "ggml-org/test-model-stories260K:F32" + draft_cache_id = "ggml-org/test-model-stories260K-infill:F32" with open(preset_path, "w") as f: f.write( "[model-dedup]\n" "hf-repo = ggml-org/test-model-stories260K\n" + "spec-draft-hf = ggml-org/test-model-stories260K-infill\n" "dedup-cache-models = 1\n" ) @@ -428,12 +450,13 @@ def test_router_dedup_cache_models(): try: ids = _get_model_ids(is_reload=False) assert "model-dedup" in ids - assert cache_id not in ids, "cache model should be hidden by dedup" + assert main_cache_id not in ids, "main cache model should be hidden by dedup" + assert draft_cache_id not in ids, "draft cache model should be hidden by dedup" # other cache models are unaffected assert "ggml-org/tinygemma3-GGUF:Q8_0" in ids # the hidden model is only hidden from the listing, it can still be used - res = server.make_request("POST", "/tokenize", data={"model": cache_id, "content": "hello"}) + res = server.make_request("POST", "/tokenize", data={"model": main_cache_id, "content": "hello"}) assert res.status_code == 200 # disabling the flag brings the cache entry back on reload @@ -441,9 +464,11 @@ def test_router_dedup_cache_models(): f.write( "[model-dedup]\n" "hf-repo = ggml-org/test-model-stories260K\n" + "spec-draft-hf = ggml-org/test-model-stories260K-infill\n" ) ids = _get_model_ids(is_reload=True) - assert cache_id in ids + assert main_cache_id in ids + assert draft_cache_id in ids # the flag also works from the global section with open(preset_path, "w") as f: @@ -453,10 +478,12 @@ def test_router_dedup_cache_models(): "\n" "[model-dedup]\n" "hf-repo = ggml-org/test-model-stories260K\n" + "spec-draft-hf = ggml-org/test-model-stories260K-infill\n" ) ids = _get_model_ids(is_reload=True) assert "model-dedup" in ids - assert cache_id not in ids, "cache model should be hidden by global dedup" + assert main_cache_id not in ids, "main cache model should be hidden by global dedup" + assert draft_cache_id not in ids, "draft cache model should be hidden by global dedup" finally: os.remove(preset_path) @@ -520,13 +547,17 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i def test_router_download_model(): - """Case 1: download a model, verify SSE events and GET /models.""" + """Case 1: download a model at the model limit, verify SSE events and GET /models.""" global server + server.models_max = 1 server.start() # Ensure the model is not present before we start server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + # A download worker must not consume or evict a model slot + _load_model_and_wait(MODEL_B, timeout=120) + sse_events: list = [] stop = threading.Event() sse_ready = threading.Event() @@ -560,6 +591,7 @@ def test_router_download_model(): # Model should now appear in GET /models ids = _get_model_ids(is_reload=False) assert MODEL_DOWNLOAD_ID in ids, f"{MODEL_DOWNLOAD_ID} not found in /models after download" + assert _get_model_status(MODEL_B) == "loaded" def test_router_delete_model(): diff --git a/tools/server/tests/unit/test_sleep.py b/tools/server/tests/unit/test_sleep.py index 515f7077d3ae..4edb2645092e 100644 --- a/tools/server/tests/unit/test_sleep.py +++ b/tools/server/tests/unit/test_sleep.py @@ -125,3 +125,43 @@ def test_server_sleep_metrics_buckets(): assert res.status_code == 200 assert is_sleeping(server) == False assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0 + + +def test_server_sleep_token_counting_wake(): + global server + server.sleep_idle_seconds = 1 + server.start() + + wait_for_sleep(server) + assert is_sleeping(server) + + res = server.make_request("POST", "/chat/completions/input_tokens", data={ + "messages": [ + {"role": "user", "content": "Hello world"} + ] + }) + assert res.status_code == 200 + assert res.body["input_tokens"] > 0 + assert is_sleeping(server) == False + + wait_for_sleep(server) + assert is_sleeping(server) + + res = server.make_request("POST", "/v1/responses/input_tokens", data={ + "input": "Hello world" + }) + assert res.status_code == 200 + assert res.body["input_tokens"] > 0 + assert is_sleeping(server) == False + + wait_for_sleep(server) + assert is_sleeping(server) + + res = server.make_request("POST", "/v1/messages/count_tokens", data={ + "messages": [ + {"role": "user", "content": "Hello world"} + ] + }) + assert res.status_code == 200 + assert res.body["input_tokens"] > 0 + assert is_sleeping(server) == False diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index 826aef2d5bcb..3a50ae5c3f48 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -155,8 +155,6 @@ def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None: else: server_path = "../../../build/bin/llama-server" server_args = [ - "--host", - self.server_host, "--port", self.server_port, "--temp", @@ -164,6 +162,7 @@ def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None: "--seed", self.seed, ] + server_args.extend(["--host", self.server_host]) if self.offline: server_args.append("--offline") if self.model_file: @@ -365,6 +364,11 @@ def stop(self) -> None: if hasattr(self, '_log') and self._log != sys.stdout: self._log.close() + def make_url(self, path: str, host: str | None = None) -> str: + if host is None: + host = self.server_host.split(",")[0].strip() + return f"http://{host}:{self.server_port}{path}" + def make_request( self, method: str, @@ -372,8 +376,9 @@ def make_request( data: dict | Any | None = None, headers: dict | None = None, timeout: float | None = DEFAULT_REQUEST_TIMEOUT, + host: str | None = None, ) -> ServerResponse: - url = f"http://{self.server_host}:{self.server_port}{path}" + url = self.make_url(path, host) parse_body = False if method == "GET": response = requests.get(url, headers=headers, timeout=timeout) @@ -407,8 +412,9 @@ def make_stream_request( path: str, data: dict | None = None, headers: dict | None = None, + host: str | None = None, ) -> Iterator[dict]: - url = f"http://{self.server_host}:{self.server_port}{path}" + url = self.make_url(path, host) if method == "POST": response = requests.post(url, headers=headers, json=data, stream=True) else: diff --git a/tools/tuning/CMakeLists.txt b/tools/tuning/CMakeLists.txt index 39ff0018026c..f07983882621 100644 --- a/tools/tuning/CMakeLists.txt +++ b/tools/tuning/CMakeLists.txt @@ -3,7 +3,7 @@ set(TARGET ggml-metal-tuning) add_executable(${TARGET} main.cpp bench.cpp fa-vec.cpp) target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT}) target_compile_features(${TARGET} PRIVATE cxx_std_17) -target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/ggml/src/ggml-metal) +target_include_directories(${TARGET} PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src/ggml-metal) if(LLAMA_TOOLS_INSTALL) install(TARGETS ${TARGET} RUNTIME) diff --git a/tools/tuning/README.md b/tools/tuning/README.md index 79e0c3646b26..b5c3586dbf1c 100644 --- a/tools/tuning/README.md +++ b/tools/tuning/README.md @@ -26,10 +26,13 @@ Sweep the grid (6 dtypes x 10 head sizes x 4 KV depths x 9 batch widths; a few h ./build/bin/ggml-metal-tuning fa-vec > fa_vec_rows.txt 2> fa_vec_sweep.log ``` -`fa_vec_rows.txt` holds nothing but table rows, ready to paste into `fa_vec_tuned_table`: the min-max-regret target, the aggregate benefit gate, the short-KV drop and the pointwise compression are already applied. +`fa_vec_rows.txt` holds nothing but table rows: the min-max-regret target, the aggregate benefit gate, the short-KV drop and the pointwise compression are already applied. +The rows carry the SKU token the runtime reported, but `fa_vec_tuned_table` is keyed by Apple GPU family, so that column has to be retagged before the rows compile. +Your family number is on the `MTLGPUFamilyApple<N>` line the backend logs at init, near the top of `fa_vec_sweep.log`; `N` is the value, and the `MTLGPUFamilyCommon`/`MTLGPUFamilyMetal` lines beside it are not it. +If your log is the only sweep for that family, its rows become the family's segment; where the family already has rows, post the log and let the two be compared before anything is replaced. A config represents a bucket only if it is no slower than the baseline config at every point that bucket covers, so a config that wins on average but loses at one batch width leaves its bucket at baseline. `fa_vec_sweep.log` holds the per-cell timings, bucket coverage, noise floor, any cooldown activity, and every config the no-harm rule refused together with the point that refused it. -Post both: the log is what makes the rows reviewable. +Post both, always: the rows now speak for every device in the family, so the log is what makes them reviewable. Long sweeps can be split. `--dtype f16,q4_0` and `--dk 128,192` restrict the grid, and the emitted rows for one `(dtype, head size)` do not depend on the others. diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cpp index f904379695ea..3d6cbeb2c1e4 100644 --- a/tools/tuning/fa-vec.cpp +++ b/tools/tuning/fa-vec.cpp @@ -56,7 +56,7 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) { ggml_set_name(m, "m"); ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f / sqrtf((float) s.dk), 0.0f, 0.0f); - ggml_flash_attn_ext_set_prec(out, GGML_PREC_F32); + ggml_prec_set_acc(out, GGML_PREC_F32); ggml_set_name(out, "out"); return out; diff --git a/tools/ui/CMakeLists.txt b/tools/ui/CMakeLists.txt index 208b46a5c15a..79ffe9fc1718 100644 --- a/tools/ui/CMakeLists.txt +++ b/tools/ui/CMakeLists.txt @@ -36,60 +36,11 @@ endif() set(UI_CPP "${CMAKE_CURRENT_BINARY_DIR}/ui.cpp") set(UI_H "${CMAKE_CURRENT_BINARY_DIR}/ui.h") -if(CMAKE_CROSSCOMPILING) - find_program(HOST_CXX_COMPILER NAMES g++ clang++ NO_CMAKE_FIND_ROOT_PATH) - if(NOT HOST_CXX_COMPILER) - message(FATAL_ERROR "UI: no host C++ compiler (g++/clang++) found to build llama-ui-embed; set -DHOST_CXX_COMPILER=<path>") - endif() - message(STATUS "UI: building llama-ui-embed with host compiler ${HOST_CXX_COMPILER}") - - if(CMAKE_HOST_WIN32) - set(LLAMA_UI_EMBED_EXE "${CMAKE_CURRENT_BINARY_DIR}/llama-ui-embed-host.exe") - else() - set(LLAMA_UI_EMBED_EXE "${CMAKE_CURRENT_BINARY_DIR}/llama-ui-embed-host") - endif() - - add_custom_command( - OUTPUT "${LLAMA_UI_EMBED_EXE}" - COMMAND "${HOST_CXX_COMPILER}" -O2 -std=c++17 - -o "${LLAMA_UI_EMBED_EXE}" "${CMAKE_CURRENT_SOURCE_DIR}/embed.cpp" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/embed.cpp" - COMMENT "Building llama-ui-embed (host)" - VERBATIM - ) - - # phony target to tie it into the dependency graph - add_custom_target(llama-ui-embed DEPENDS "${LLAMA_UI_EMBED_EXE}") -else() - # exclude llama-ui-embed from sanitizer flags, - # it's a build-time-only tool, no need to instrument it - # this is to fix TSan "memory layout is incompatible" error on CI - get_directory_property(_llama_ui_dir_co COMPILE_OPTIONS) - get_directory_property(_llama_ui_dir_ll LINK_LIBRARIES) - set(_llama_ui_embed_co ${_llama_ui_dir_co}) - set(_llama_ui_embed_ll ${_llama_ui_dir_ll}) - list(FILTER _llama_ui_embed_co EXCLUDE REGEX ".*-fsanitize=.*") - list(FILTER _llama_ui_embed_ll EXCLUDE REGEX ".*-fsanitize=.*") - set_directory_properties(PROPERTIES - COMPILE_OPTIONS "${_llama_ui_embed_co}" - LINK_LIBRARIES "${_llama_ui_embed_ll}") - - add_executable(llama-ui-embed embed.cpp) - target_compile_features(llama-ui-embed PRIVATE cxx_std_17) - set_target_properties(llama-ui-embed PROPERTIES - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" - ) - set(LLAMA_UI_EMBED_EXE "$<TARGET_FILE:llama-ui-embed>") - - # restore so the llama-ui library below keeps sanitizer instrumentation - set_directory_properties(PROPERTIES - COMPILE_OPTIONS "${_llama_ui_dir_co}" - LINK_LIBRARIES "${_llama_ui_dir_ll}") -endif() - -# Run the provisioning script every build so source changes in tools/ui/ are -# always picked up. The script uses copy_if_different for ui.cpp/ui.h, so the -# library only recompiles when contents actually change. +# Provision assets and generate ui.cpp/ui.h natively in CMake at build time. +# The generated sources are compiled by the regular target toolchain; no +# build-time host executable is needed (works in any cross-compile setup). +# The script uses copy_if_different semantics, so the library below only +# recompiles when the generated contents actually change. add_custom_target(llama-ui-assets ALL BYPRODUCTS ${UI_CPP} ${UI_H} COMMAND ${CMAKE_COMMAND} @@ -101,15 +52,12 @@ add_custom_target(llama-ui-assets ALL "-DHF_VERSION=${HF_UI_VERSION}" "-DHF_ENABLED=${LLAMA_USE_PREBUILT_UI}" "-DBUILD_UI=${LLAMA_BUILD_UI}" - "-DLLAMA_UI_EMBED=${LLAMA_UI_EMBED_EXE}" "-DLLAMA_UI_GZIP=${LLAMA_UI_GZIP}" -P "${PROJECT_SOURCE_DIR}/scripts/ui-assets.cmake" COMMENT "Provisioning UI assets" VERBATIM ) -add_dependencies(llama-ui-assets llama-ui-embed) - set_source_files_properties(${UI_CPP} ${UI_H} PROPERTIES GENERATED TRUE) add_library(${TARGET} STATIC ${UI_CPP} ${UI_H}) diff --git a/tools/ui/embed.cpp b/tools/ui/embed.cpp deleted file mode 100644 index b76c9047f289..000000000000 --- a/tools/ui/embed.cpp +++ /dev/null @@ -1,308 +0,0 @@ -// llama-ui-embed: generate ui.cpp / ui.h that embed UI assets as C arrays. -// -// Usage: -// llama-ui-embed <out_cpp> <out_h> [<asset_dir>] -// -// Recursively embeds every regular file under <asset_dir>. -// Asset names are relative paths from <asset_dir> (e.g. "_app/immutable/bundle.HASH.js"). -// Without <asset_dir>, emits an empty asset table. - -#include <inttypes.h> -#include <stdarg.h> -#include <stdint.h> -#include <stdio.h> -#include <string.h> - -#include <algorithm> -#include <filesystem> -#include <fstream> -#include <functional> -#include <string> -#include <vector> - - -static const char * mime_from_ext(const std::string & name) { - auto ext = name.rfind('.'); - if (ext == std::string::npos) return "application/octet-stream"; - std::string e = name.substr(ext + 1); - if (e == "html") return "text/html; charset=utf-8"; - if (e == "css") return "text/css"; - if (e == "js") return "application/javascript"; - if (e == "json") return "application/json"; - if (e == "webmanifest") return "application/manifest+json"; - if (e == "svg") return "image/svg+xml"; - if (e == "png") return "image/png"; - if (e == "jpg" || - e == "jpeg") return "image/jpeg"; - if (e == "ico") return "image/x-icon"; - if (e == "woff") return "font/woff"; - if (e == "woff2") return "font/woff2"; - return "application/octet-stream"; -} - -// Computes FNV-1a hash of the data -static uint64_t fnv_hash(const uint8_t * data, size_t len) { - const uint64_t fnv_prime = 0x100000001b3ULL; - uint64_t hash = 0xcbf29ce484222325ULL; - - for (size_t i = 0; i < len; ++i) { - hash ^= data[i]; - hash *= fnv_prime; - } - return hash; -} - -static bool read_file(const std::filesystem::path & path, std::vector<unsigned char> & out) { - std::ifstream f(path, std::ios::binary | std::ios::ate); - if (!f) { - fprintf(stderr, "embed: cannot open %s\n", path.string().c_str()); - return false; - } - const auto sz = f.tellg(); - if (sz < 0) { - return false; - } - f.seekg(0); - out.resize(static_cast<size_t>(sz)); - if (sz > 0 && !f.read(reinterpret_cast<char *>(out.data()), sz)) { - return false; - } - return true; -} - -static void append_bytes_hex(std::string & out, const std::vector<unsigned char> & bytes) { - static const char hex[] = "0123456789abcdef"; - out.reserve(out.size() + bytes.size() * 5); - for (unsigned char b : bytes) { - out += '0'; - out += 'x'; - out += hex[b >> 4]; - out += hex[b & 0xf]; - out += ','; - } -} - -static bool write_if_different(const std::string & path, const std::string & content) { - std::ifstream f(path, std::ios::binary | std::ios::ate); - if (f) { - const auto sz = f.tellg(); - if (sz >= 0 && static_cast<size_t>(sz) == content.size()) { - std::string existing(static_cast<size_t>(sz), '\0'); - f.seekg(0); - if (sz == 0 || f.read(existing.data(), sz)) { - if (existing == content) { - return true; - } - } - } - } - - std::ofstream out(path, std::ios::binary | std::ios::trunc); - if (!out) { - fprintf(stderr, "embed: cannot write %s\n", path.c_str()); - return false; - } - if (!content.empty()) { - out.write(content.data(), static_cast<std::streamsize>(content.size())); - } - bool ok = out.good(); - if (ok) { - printf("embed: write output file %s\n", path.c_str()); - } - return ok; -} - -static std::string path_basename(const std::string & name) { - const size_t p = name.rfind('/'); - return p == std::string::npos ? name : name.substr(p + 1); -} -static bool str_starts_with(const std::string & s, const char * prefix) { - const size_t n = strlen(prefix); - return s.size() >= n && s.compare(0, n, prefix) == 0; -} -static bool str_ends_with(const std::string & s, const char * suffix) { - const size_t n = strlen(suffix); - return s.size() >= n && s.compare(s.size() - n, n, suffix) == 0; -} - -static std::string fmt(const char * pattern, ...) { - char tmp[512]; - va_list ap; - va_start(ap, pattern); - const int n = vsnprintf(tmp, sizeof(tmp), pattern, ap); - va_end(ap); - return (n > 0) ? std::string(tmp, static_cast<size_t>(n)) : std::string(); -} - -struct asset_entry { - std::string name; - std::filesystem::path path; -}; - -int main(int argc, char ** argv) { - if (argc < 3 || argc > 4) { - fprintf(stderr, "usage: %s <out_cpp> <out_h> [<asset_dir>]\n", argv[0]); - return 1; - } - - const std::string out_cpp = argv[1]; - const std::string out_h = argv[2]; - const std::string asset_dir = (argc >= 4) ? argv[3] : std::string(); - - const bool use_gzip = !asset_dir.empty() && std::filesystem::exists(asset_dir + "/_gzip"); - const std::string in_dir = use_gzip ? (asset_dir + "/_gzip") : asset_dir; - - std::vector<asset_entry> assets; - if (!in_dir.empty()) { - const std::filesystem::path dir = in_dir; - - std::error_code ec; - std::filesystem::recursive_directory_iterator it(dir, ec); - if (ec) { - fprintf(stderr, "embed: cannot iterate %s: %s\n", argv[3], ec.message().c_str()); - return 1; - } - for (const auto & entry : it) { - if (!entry.is_regular_file()) { - continue; - } - // name is the relative path from dir, with forward slashes - const std::string name = entry.path().lexically_relative(dir).generic_string(); - assets.push_back({ name, entry.path() }); - } - - // directory iteration order is unspecified; sort for reproducible output - std::sort(assets.begin(), assets.end(), - [](const asset_entry & a, const asset_entry & b) { return a.name < b.name; }); - } - - const int n_assets = static_cast<int>(assets.size()); - - if (n_assets > 0) { - using match_fn = std::function<bool(const std::string &)>; - auto exact = [](const char * name) -> match_fn { - return [name](const std::string & base) { return base == name; }; - }; - - struct required_check { const char * label; match_fn match; bool found; }; - required_check checks[] = { - { "index.html", exact("index.html"), false }, - { "manifest.webmanifest", exact("manifest.webmanifest"), false }, - { "sw.js", exact("sw.js"), false }, - { "build.json", exact("build.json"), false }, - { "version.json", exact("version.json"), false }, - { "bundle[hash].js", [](const std::string & b) { - return str_starts_with(b, "bundle") && str_ends_with(b, ".js"); - }, false }, - { "bundle[hash].css", [](const std::string & b) { - return str_starts_with(b, "bundle") && str_ends_with(b, ".css"); - }, false }, - { "workbox[hash].js", [](const std::string & b) { - return str_starts_with(b, "workbox") && str_ends_with(b, ".js"); - }, false }, - }; - - for (const auto & a : assets) { - const std::string base = path_basename(a.name); - for (auto & c : checks) { - if (!c.found) { c.found = c.match(base); } - } - } - - std::vector<const char *> missing; - for (const auto & c : checks) { - if (!c.found) { missing.push_back(c.label); } - } - if (!missing.empty()) { - fprintf(stderr, "\ncurrent asset files:\n"); - for (const auto & a : assets) { - fprintf(stderr, " %s\n", a.name.c_str()); - } - fprintf(stderr, "missing required asset(s):\n"); - for (const char * m : missing) { - fprintf(stderr, " %s\n", m); - } - fprintf(stderr, "hint: try cleaning your build directory: %s\n", in_dir.c_str()); - return 1; - } - } - - std::string h; - h += "#pragma once\n\n#include <array>\n#include <string>\n\n"; - if (n_assets > 0) { - h += "#define LLAMA_UI_HAS_ASSETS 1\n\n"; - } - h += - "struct llama_ui_asset {\n" - " std::string name;\n" - " const unsigned char * data;\n" - " std::size_t size;\n" - " std::string etag;\n" - " std::string type;\n" - "};\n\n" - "const llama_ui_asset * llama_ui_find_asset(const std::string & name);\n" - "bool llama_ui_use_gzip();\n"; - h += fmt("const std::array<llama_ui_asset, %d> & llama_ui_get_assets();\n", n_assets); - - std::string cpp; - cpp += "#include \"ui.h\"\n\n"; - - if (n_assets > 0) { - for (int i = 0; i < n_assets; i++) { - std::vector<unsigned char> bytes; - if (!read_file(assets[i].path, bytes)) { - return 1; - } - if (bytes.empty()) { - fprintf(stderr, "embed: empty file: %s\n", assets[i].path.generic_string().c_str()); - return 1; - } - cpp += fmt("static const unsigned char asset_%d_data[] = {", i); - append_bytes_hex(cpp, bytes); - - // note: this is a simple hash for cache busting, not a cryptographic hash; fnv is enough here - const auto hash = fnv_hash(bytes.data(), bytes.size()); - - cpp += fmt("};\nstatic const std::size_t asset_%d_size = %zu;\n", - i, bytes.size()); - cpp += fmt("static const char asset_%d_etag[] = \"\\\"0x%016" PRIx64 "\\\"\";\n\n", - i, hash); - } - - cpp += fmt("static const std::array<llama_ui_asset, %d> g_assets = {{\n", n_assets); - for (int i = 0; i < n_assets; i++) { - const std::string & name = assets[i].name; - cpp += fmt(" { \"%s\", asset_%d_data, asset_%d_size, asset_%d_etag, \"%s\" },\n", - name.c_str(), i, i, i, mime_from_ext(name)); - } - cpp += "}};\n\n"; - - cpp += - "const llama_ui_asset * llama_ui_find_asset(const std::string & name) {\n" - " for (const auto & a : g_assets) {\n" - " if (a.name == name) {\n" - " return &a;\n" - " }\n" - " }\n" - " return nullptr;\n" - "}\n"; - cpp += fmt("const std::array<llama_ui_asset, %d> & llama_ui_get_assets() {\n", n_assets); - cpp += " return g_assets;\n" - "}\n"; - } else { - cpp += - "const llama_ui_asset * llama_ui_find_asset(const std::string &) {\n" - " return nullptr;\n" - "}\n" - "const std::array<llama_ui_asset, 0> & llama_ui_get_assets() {\n" - " static const std::array<llama_ui_asset, 0> empty{};\n" - " return empty;\n" - "}\n"; - } - cpp += fmt("bool llama_ui_use_gzip() { return %s; }\n", use_gzip ? "true" : "false"); - - bool ok = true; - ok = write_if_different(out_h, h) && ok; - ok = write_if_different(out_cpp, cpp) && ok; - return ok ? 0 : 1; -} diff --git a/tools/ui/src/app.d.ts b/tools/ui/src/app.d.ts index 5309dce8f4dc..639a16df215d 100644 --- a/tools/ui/src/app.d.ts +++ b/tools/ui/src/app.d.ts @@ -137,7 +137,6 @@ declare global { declare global { interface Window { - idxThemeStyle?: number; idxCodeBlock?: number; // File System Access API - not in the DOM lib and unavailable in some browsers diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte index 05bd733a2cdc..754856d60b86 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte @@ -41,6 +41,9 @@ }: Props = $props(); const scrollClasses = $derived(limitToSingleRow ? 'first:ml-4 last:mr-4' : ''); + // Carousel items must keep their width; wrapped attachments (message bubbles) + // shrink so wide images fit the bubble instead of overflowing it + const layoutClasses = $derived(limitToSingleRow ? 'flex-shrink-0' : 'min-w-0'); function toMcpResourceAttachment( extra: DatabaseMessageExtraMcpResource, @@ -92,7 +95,7 @@ /> {:else if item.isImage && item.preview} <ChatAttachmentsListItemThumbnailImage - class="flex-shrink-0 cursor-pointer {className} {scrollClasses}" + class="{layoutClasses} cursor-pointer {className} {scrollClasses}" height={imageHeight} id={item.id} {imageClass} diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailImage.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailImage.svelte index 34db43339232..79a8f115b336 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailImage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailImage.svelte @@ -34,7 +34,7 @@ {/snippet} <div - class="group relative overflow-hidden rounded-lg bg-muted shadow-lg dark:border dark:border-muted {className}" + class="group relative min-w-0 overflow-hidden rounded-lg bg-muted shadow-lg dark:border dark:border-muted {className}" > {#if onclick} <button diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte index 9c3a9e89122a..1ce731845fe7 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte @@ -1,6 +1,10 @@ <script lang="ts"> import { File, Image, MessageSquare, Mic, Plus, Video } from '@lucide/svelte'; - import { ChatFormActionAddToolsSubmenu, McpLogo } from '$lib/components/app'; + import { + ChatFormActionAddReasoningSubmenu, + ChatFormActionAddToolsSubmenu, + McpLogo + } from '$lib/components/app'; import { buttonVariants } from '$lib/components/ui/button'; import * as DropdownMenu from '$lib/components/ui/dropdown-menu'; import * as Tooltip from '$lib/components/ui/tooltip'; @@ -13,6 +17,7 @@ import { getChatFormActionsContext } from '$lib/contexts'; import { AttachmentAction, AttachmentItemEnabledWhen } from '$lib/enums'; import { useAttachmentMenu } from '$lib/hooks/use-attachment-menu.svelte'; + import { serverStore } from '$lib/stores'; interface Props { class?: string; @@ -92,6 +97,13 @@ } }} > + <!-- in router mode the models selector owns the reasoning submenu --> + {#if !serverStore.isRouterMode} + <ChatFormActionAddReasoningSubmenu /> + + <DropdownMenu.Separator /> + {/if} + <DropdownMenu.Item class="flex cursor-pointer items-center gap-2" onclick={() => attachmentMenu.callbacks[AttachmentAction.FILE_UPLOAD]()} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte index fa2a50bc5cbd..46d05338b619 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte @@ -404,7 +404,7 @@ } </script> -<div class:chat-message--synthetic={isSynthetic} class="chat-message"> +<div> {#if message.role === MessageRole.SYSTEM} <ChatMessageSystem bind:textareaElement class={className} {message} /> {:else if mcpPromptExtra} @@ -425,25 +425,3 @@ /> {/if} </div> - -<style> - /* - * The browser skips layout and paint for messages outside the - * viewport. contain-intrinsic-size reuses the last rendered size - * once known; 500px sizes messages that have never been rendered. - */ - .chat-message { - --chat-message-intrinsic-size: 500px; - content-visibility: auto; - contain-intrinsic-size: auto var(--chat-message-intrinsic-size); - } - - /* - * Synthetic rows (e.g. the working-directory change) are small, so an - * accurate placeholder keeps the injected row from inflating the - * auto-scroll offset; the 500px default is for ordinary bubbles. - */ - .chat-message--synthetic { - --chat-message-intrinsic-size: 40px; - } -</style> diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte index a2c742f0fbf2..dac55caff074 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte @@ -82,8 +82,11 @@ let lastUserMessageHeight = $state(0); let assistantMarginTop = $state(0); + // The measured CSS vars feed the :last-child min-height rule only, so only + // the last assistant message needs them. Reading isLastAssistantMessage + // here also re-runs the effect when this message stops being the last. $effect(() => { - if (!assistantEl) return; + if (!assistantEl || !isLastAssistantMessage) return; assistantMarginTop = Math.round(parseFloat(getComputedStyle(assistantEl).marginTop)); diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte index a604a97e39ee..6b0b573288d1 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte @@ -13,7 +13,12 @@ import ChatMessageToolCallBlockWriteFile from './ChatMessageToolCallBlockWriteFile.svelte'; import { BuiltInTool } from '$lib/enums'; import type { AgenticSection, DatabaseMessageExtra } from '$lib/types'; - import { extractSearchQuery, extractSearchResults, isWebSearchToolName } from '$lib/utils'; + import { + extractSearchQuery, + extractSearchResults, + isWebSearchToolName, + looksLikeSearchResult + } from '$lib/utils'; interface Props { section: AgenticSection; @@ -26,11 +31,16 @@ let { attachments, isExecuting, isStreaming, onToggle, open, section }: Props = $props(); - const searchResults = $derived(extractSearchResults(section.toolResult)); - const searchQuery = $derived(extractSearchQuery(section.toolArgs)); - const isSearchCall = $derived( - searchResults.length > 0 || (searchQuery.length > 0 && isWebSearchToolName(section.toolName)) - ); + // Runs for every tool block on mount, before the body renders: the cheap + // content prefilter and the tool-name allow-list come first so blobs from + // exec/file tools are never line-split or JSON-parsed here + const isSearchCall = $derived.by(() => { + if (looksLikeSearchResult(section.toolResult)) { + return extractSearchResults(section.toolResult).length > 0; + } + + return isWebSearchToolName(section.toolName) && extractSearchQuery(section.toolArgs).length > 0; + }); </script> {#if isSearchCall} @@ -38,7 +48,7 @@ {:else if section.toolName === BuiltInTool.BROWSER_GET_DATETIME} <ChatMessageToolCallBlockGetDatetime {isStreaming} {section} /> {:else if section.toolName === BuiltInTool.SERVER_GET_INFO} - <ChatMessageToolCallBlockGetInfo {isStreaming} {section} /> + <ChatMessageToolCallBlockGetInfo {isStreaming} {onToggle} {open} {section} /> {:else if section.toolName === BuiltInTool.SERVER_READ_FILE} <ChatMessageToolCallBlockReadFile {isStreaming} {onToggle} {open} {section} /> {:else if section.toolName === BuiltInTool.BROWSER_READ_MEDIA} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte index 2067e4268868..e187ec640532 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte @@ -1,5 +1,5 @@ <script lang="ts"> - import { parseEditFileMeta } from './parsers/edit-file'; + import { parseEditFileMeta, parseEditFileTitleMeta } from './parsers/edit-file'; import ToolCallBlock from './ToolCallBlock.svelte'; import { XCircle } from '@lucide/svelte'; import { MAX_HEIGHT_CODE_BLOCK, RESULT_STAT_SEPARATOR } from '$lib/constants'; @@ -16,24 +16,32 @@ let { isStreaming, onToggle, open, section }: Props = $props(); - const editFileMeta = $derived(parseEditFileMeta(section)); + const editFileMeta = $derived(parseEditFileTitleMeta(section)); + // body-only: the full meta parses the embedded edit strings, and these + // deriveds are read solely from the children snippet, which renders only + // while the block is expanded + const editFileBody = $derived(parseEditFileMeta(section)); const home = $derived(toolsStore.serverHome); const editDiffs = $derived( - (editFileMeta?.edits ?? []).map((edit) => computeLineDiff(edit.oldText, edit.newText)) + (editFileBody?.edits ?? []).map((edit) => computeLineDiff(edit.oldText, edit.newText)) ); </script> <ToolCallBlock {isStreaming} meta={editFileMeta} {onToggle} {open} {section}> {#snippet titleSnippet()} - <span class="text-muted-foreground">Edit file </span> + <span class="flex min-w-0 flex-wrap items-baseline gap-x-1"> + <span class="shrink-0 text-muted-foreground">Edit file</span> - <span class="font-mono" title={editFileMeta?.filePath} - >{abbreviateHome(editFileMeta?.filePath ?? '', home)}</span - > + <span class="flex min-w-0 items-baseline gap-1.5"> + <span class="min-w-0 overflow-x-auto font-mono" title={editFileMeta?.filePath}> + {abbreviateHome(editFileMeta?.filePath ?? '', home)} + </span> - {#if editFileMeta?.errorMessage} - <span class="ml-1 text-xs italic text-muted-foreground/70">(failed)</span> - {/if} + {#if editFileMeta?.errorMessage} + <span class="shrink-0 text-xs italic text-muted-foreground/70">(failed)</span> + {/if} + </span> + </span> {/snippet} {#snippet children(meta, _ctx)} @@ -45,11 +53,11 @@ <span>{meta.errorMessage}</span> </div> - {:else if meta && meta.edits.length > 0} + {:else if meta && editFileBody && editFileBody.edits.length > 0} {#each editDiffs as diffLines, ei (ei)} <div class={ei === 0 ? '' : 'mt-3'}> <div class="mb-1.5 text-xs text-muted-foreground/70 italic"> - Edit {ei + 1} of {meta.edits.length} + Edit {ei + 1} of {editFileBody.edits.length} </div> <div style:max-height={MAX_HEIGHT_CODE_BLOCK} class="diff-block"> diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockGetInfo.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockGetInfo.svelte index bd46b76dc96a..225d9def90a4 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockGetInfo.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockGetInfo.svelte @@ -1,20 +1,18 @@ <script lang="ts"> - import { Info, Loader2 } from '@lucide/svelte'; - import { AgenticSectionType } from '$lib/enums'; + import ToolCallBlock from './ToolCallBlock.svelte'; + import { XCircle } from '@lucide/svelte'; import { toolsStore } from '$lib/stores'; import type { AgenticSection } from '$lib/types'; import { abbreviateHome } from '$lib/utils'; interface Props { section: AgenticSection; - isStreaming?: boolean; + open: boolean; + isStreaming: boolean; + onToggle?: () => void; } - let { isStreaming = false, section }: Props = $props(); - - const isPending = $derived(section.type === AgenticSectionType.TOOL_CALL_PENDING); - const isStreamingCall = $derived(section.type === AgenticSectionType.TOOL_CALL_STREAMING); - const showSpinner = $derived(isPending || (isStreamingCall && isStreaming)); + let { isStreaming, onToggle, open, section }: Props = $props(); type GetInfoMeta = { os?: string; @@ -50,29 +48,79 @@ const cwdDisplay = $derived(abbreviateHome(infoMeta.cwd ?? '', home)); </script> -<div class="text-muted-foreground flex items-center gap-2 py-1.5"> - <Info class="text-muted-foreground/60 h-3.5 w-3.5 shrink-0" /> - - {#if showSpinner} - <span class="text-foreground/80 text-sm font-medium">Runtime info</span> - - <Loader2 class="text-muted-foreground/70 h-3 w-3 animate-spin" /> - {:else if infoMeta.errorMessage} - <span class="text-foreground/80 text-sm font-medium">Runtime info </span> - - <span class="text-red-600 text-xs italic dark:text-red-400">- {infoMeta.errorMessage}</span - > - {:else if infoMeta.os || infoMeta.cwd} - <span class="text-foreground/80 text-sm font-medium">Runtime info </span> - - {#if infoMeta.os} - <span class="font-mono text-foreground/90 text-sm">{infoMeta.os}</span> - {/if} - - {#if infoMeta.cwd} - <span class="font-mono text-foreground/90 text-sm" title={infoMeta.cwd}>{cwdDisplay}</span> +<ToolCallBlock + {isStreaming} + meta={infoMeta} + {onToggle} + {open} + {section} + spinIconWhenActive + title="Runtime info" +> + {#snippet children(meta, _ctx)} + {#if meta?.errorMessage} + <div + class="flex items-start gap-2 rounded bg-red-500/10 p-2 text-xs text-red-600 italic dark:text-red-400" + > + <XCircle class="mt-0.5 h-3 w-3 shrink-0" /> + + <span>{meta.errorMessage}</span> + </div> + {:else if infoMeta.os || infoMeta.cwd} + <table class="w-full table-fixed border-collapse text-sm"> + <colgroup> + <col class="w-12" /> + + <col /> + </colgroup> + + <tbody class="divide-y divide-border/50"> + {#if infoMeta.os} + <tr> + <th + class="py-1 pr-3 text-left align-baseline text-[11px] font-medium tracking-wide text-muted-foreground/60 uppercase" + scope="row" + > + os + </th> + + <td class="py-1 align-baseline"> + <div class="min-w-0 overflow-x-auto font-mono text-foreground/90"> + {infoMeta.os} + </div> + </td> + </tr> + {/if} + + {#if infoMeta.cwd} + <tr> + <th + class="py-1 pr-3 text-left align-baseline text-[11px] font-medium tracking-wide text-muted-foreground/60 uppercase" + scope="row" + > + cwd + </th> + + <td class="py-1 align-baseline"> + <div + class="min-w-0 overflow-x-auto font-mono text-foreground/90" + title={infoMeta.cwd} + > + {cwdDisplay} + </div> + </td> + </tr> + {/if} + </tbody> + </table> + {:else if section.toolResult} + <div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic"> + {section.toolResult} + </div> + {:else} + <div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic"> + Waiting for runtime info... + </div> {/if} - {:else} - <span class="text-foreground/80 text-sm font-medium">Runtime info</span> - {/if} -</div> + {/snippet} +</ToolCallBlock> diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadFile.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadFile.svelte index 13b44022282f..1134c8c0fecf 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadFile.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadFile.svelte @@ -19,15 +19,19 @@ <ToolCallBlock {isStreaming} meta={readFileMeta} {onToggle} {open} {section}> {#snippet titleSnippet()} - <span class="text-muted-foreground">Read file </span> + <span class="flex min-w-0 flex-wrap items-baseline gap-x-1"> + <span class="shrink-0 text-muted-foreground">Read file</span> - <span class="font-mono">{readFileMeta?.fileName}</span> + <span class="flex min-w-0 items-baseline gap-1.5"> + <span class="min-w-0 overflow-x-auto font-mono">{readFileMeta?.fileName}</span> - {#if readFileMeta?.lineRange} - <span class="text-muted-foreground" - > (lines {readFileMeta.lineRange.start}-{readFileMeta.lineRange.end})</span - > - {/if} + {#if readFileMeta?.lineRange} + <span class="shrink-0 text-muted-foreground"> + (lines {readFileMeta.lineRange.start}-{readFileMeta.lineRange.end}) + </span> + {/if} + </span> + </span> {/snippet} {#snippet children(_meta, _ctx)} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte index 93d8990184d1..0948bf623b89 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte @@ -45,9 +45,11 @@ <ToolCallBlock {isStreaming} meta={readMediaMeta} {onToggle} {open} {section}> {#snippet titleSnippet()} - <span class="text-muted-foreground">Read media </span> + <span class="flex min-w-0 flex-wrap items-baseline gap-x-1"> + <span class="shrink-0 text-muted-foreground">Read media</span> - <span class="font-mono">{readMediaMeta?.fileName}</span> + <span class="min-w-0 overflow-x-auto font-mono">{readMediaMeta?.fileName}</span> + </span> {/snippet} {#snippet children(_meta, _ctx)} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockWriteFile.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockWriteFile.svelte index 178c479d98f7..ac2a8e6576c6 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockWriteFile.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockWriteFile.svelte @@ -1,5 +1,5 @@ <script lang="ts"> - import { parseWriteFileMeta } from './parsers/write-file'; + import { parseWriteFileMeta, parseWriteFileTitleMeta } from './parsers/write-file'; import ToolCallBlock from './ToolCallBlock.svelte'; import { XCircle } from '@lucide/svelte'; import { SyntaxHighlightedCode } from '$lib/components/app'; @@ -17,21 +17,29 @@ let { isStreaming, onToggle, open, section }: Props = $props(); - const writeFileMeta = $derived(parseWriteFileMeta(section)); + const writeFileMeta = $derived(parseWriteFileTitleMeta(section)); + // body-only: the full meta parses the embedded file content, and this + // derived is read solely from the children snippet, which renders only + // while the block is expanded + const writeFileBody = $derived(parseWriteFileMeta(section)); const home = $derived(toolsStore.serverHome); </script> <ToolCallBlock {isStreaming} meta={writeFileMeta} {onToggle} {open} {section}> {#snippet titleSnippet()} - <span class="text-muted-foreground">Write file </span> + <span class="flex min-w-0 flex-wrap items-baseline gap-x-1"> + <span class="shrink-0 text-muted-foreground">Write file</span> - <span class="font-mono" title={writeFileMeta?.filePath} - >{abbreviateHome(writeFileMeta?.filePath ?? '', home)}</span - > + <span class="flex min-w-0 items-baseline gap-1.5"> + <span class="min-w-0 overflow-x-auto font-mono" title={writeFileMeta?.filePath}> + {abbreviateHome(writeFileMeta?.filePath ?? '', home)} + </span> - {#if writeFileMeta?.errorMessage} - <span class="ml-1 text-xs italic text-muted-foreground/70">(failed)</span> - {/if} + {#if writeFileMeta?.errorMessage} + <span class="shrink-0 text-xs italic text-muted-foreground/70">(failed)</span> + {/if} + </span> + </span> {/snippet} {#snippet children(meta, ctx)} @@ -45,7 +53,7 @@ </div> {:else if meta} <SyntaxHighlightedCode - code={meta.content} + code={writeFileBody?.content ?? ''} language={meta.language} maxHeight={MAX_HEIGHT_CODE_BLOCK} streaming={ctx.isCodeStreaming} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/_shared.ts b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/_shared.ts index 073e03de27b1..79d7c2e27e6a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/_shared.ts +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/_shared.ts @@ -4,6 +4,7 @@ // args-present check, JSON parse) - keeping them here lets each parser // stay focused on its own format quirks. +import { TOOL_ARG_STRING_FIELD_PATTERN_TEMPLATE } from '$lib/constants'; import { BuiltInTool } from '$lib/enums'; import type { AgenticSection } from '$lib/types/agentic'; import { parsePartialJsonArgs } from '$lib/utils/parse-partial-json-args'; @@ -28,6 +29,45 @@ function parseFinalToolArgs(blob: string): Record<string, unknown> | null { } } +// Compiled per key on first use; the key set is tiny and fixed. +const toolArgStringRegexes = new Map<string, RegExp>(); + +/** + * Extract a string field from a JSON tool-args blob without parsing the + * whole document. write_file and edit_file args embed full file contents, + * yet the block title needs only the path; a targeted key match plus a + * JSON.parse of the captured string literal alone keeps title rendering + * O(path) instead of O(blob). Returns undefined when the key is missing + * or its value is not a string; callers fall back to the full parse. + */ +export function extractToolArgString( + toolArgs: string, + keys: readonly string[] +): string | undefined { + for (const key of keys) { + let pattern = toolArgStringRegexes.get(key); + + if (!pattern) { + pattern = new RegExp(TOOL_ARG_STRING_FIELD_PATTERN_TEMPLATE.replace('{key}', key)); + toolArgStringRegexes.set(key, pattern); + } + + const match = pattern.exec(toolArgs); + + if (!match) continue; + + try { + const value: unknown = JSON.parse(`"${match[1]}"`); + + if (typeof value === 'string') return value; + } catch { + // fall through to the next key; the full parse is the fallback + } + } + + return undefined; +} + /** * Parse a section's toolArgs against an expected tool name. Returns * `null` when: diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/edit-file.ts b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/edit-file.ts index 9ed6f92bc089..d711466cb27c 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/edit-file.ts +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/edit-file.ts @@ -3,26 +3,12 @@ // rendering), plus the result blob for `result` / `edits_applied` / // `error` fields. -import { parseToolArgs } from './_shared'; -import { FILE_PATH_SEPARATOR_REGEX } from '$lib/constants'; +import { extractToolArgString, parseToolArgs } from './_shared'; +import { FILE_PATH_SEPARATOR_REGEX, TOOL_ARG_PATH_KEYS } from '$lib/constants'; import { BuiltInTool } from '$lib/enums'; -import type { AgenticSection } from '$lib/types'; +import type { AgenticSection, EditFileEdit, EditFileMeta, EditFileTitleMeta } from '$lib/types'; import { tryParseToolResultObject } from '$lib/utils'; -export type EditFileEdit = { - oldText: string; - newText: string; -}; - -export type EditFileMeta = { - fileName: string; - filePath: string; - edits: EditFileEdit[]; - resultMessage?: string; - editsApplied?: number; - errorMessage?: string; -}; - export function parseEditFileMeta(section: AgenticSection): EditFileMeta | null { const args = parseToolArgs(BuiltInTool.SERVER_EDIT_FILE, section, { partial: true }); @@ -79,3 +65,45 @@ export function parseEditFileMeta(section: AgenticSection): EditFileMeta | null resultMessage }; } + +/** + * Title-tier meta for edit_file blocks: everything the header and status + * pill render, obtained without parsing the embedded edit strings. The path + * comes from a targeted key extraction; the full parse runs only as a + * fallback for arg shapes the extraction can't see. + */ +export function parseEditFileTitleMeta(section: AgenticSection): EditFileTitleMeta | null { + if (section.toolName !== BuiltInTool.SERVER_EDIT_FILE || !section.toolArgs) return null; + + let rawPath: string | undefined = extractToolArgString(section.toolArgs, TOOL_ARG_PATH_KEYS); + + if (!rawPath) { + const args = parseToolArgs(BuiltInTool.SERVER_EDIT_FILE, section, { partial: true }); + const fallbackPath = args?.path ?? args?.file_path ?? args?.filePath; + + if (typeof fallbackPath === 'string' && fallbackPath) rawPath = fallbackPath; + } + + if (!rawPath) return null; + + const fileName = rawPath.split(FILE_PATH_SEPARATOR_REGEX).pop() || rawPath; + const resultObj = tryParseToolResultObject(section.toolResult); + + let resultMessage: string | undefined; + let editsApplied: number | undefined; + let errorMessage: string | undefined; + + if (typeof resultObj?.error === 'string') { + errorMessage = resultObj.error; + } else if (resultObj) { + if (typeof resultObj.result === 'string') { + resultMessage = resultObj.result; + } + + if (Number.isFinite(Number(resultObj.edits_applied))) { + editsApplied = Number(resultObj.edits_applied); + } + } + + return { editsApplied, errorMessage, fileName, filePath: rawPath, resultMessage }; +} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/run-javascript.ts b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/run-javascript.ts index 440a1f5d65a9..bd97cd2feb6d 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/run-javascript.ts +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/run-javascript.ts @@ -6,6 +6,7 @@ // are handled. import { parseToolArgs } from './_shared'; +import { JSON_ARRAY_OPEN, JSON_OBJECT_OPEN } from '$lib/constants'; import { BuiltInTool } from '$lib/enums'; import type { AgenticSection } from '$lib/types'; @@ -38,14 +39,21 @@ export function parseRunJavascriptMeta(section: AgenticSection): RunJavascriptMe // do we scan raw lines for the `Error:` prefix. let parsedObject: Record<string, unknown> | null = null; - try { - const parsed: unknown = JSON.parse(toolResultString); + // Successful sandbox output is a JSON array, errors are objects; plain + // text (huge console logs) fails the parse below anyway, so only try + // when the blob starts with a JSON container + const trimmedResult = toolResultString.trimStart(); - if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { - parsedObject = parsed as Record<string, unknown>; + if (trimmedResult[0] === JSON_OBJECT_OPEN || trimmedResult[0] === JSON_ARRAY_OPEN) { + try { + const parsed: unknown = JSON.parse(trimmedResult); + + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + parsedObject = parsed as Record<string, unknown>; + } + } catch { + parsedObject = null; } - } catch { - parsedObject = null; } if (typeof parsedObject?.error === 'string') { diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/write-file.ts b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/write-file.ts index 5b9bf9f88c32..4a8e1a9c980d 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/write-file.ts +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/write-file.ts @@ -3,22 +3,12 @@ // finishes) and surfaces `bytes`, `result`, and `error` from the // result blob. -import { parseToolArgs } from './_shared'; -import { CODE_BLOCK, FILE_PATH_SEPARATOR_REGEX } from '$lib/constants'; +import { extractToolArgString, parseToolArgs } from './_shared'; +import { CODE_BLOCK, FILE_PATH_SEPARATOR_REGEX, TOOL_ARG_PATH_KEYS } from '$lib/constants'; import { BuiltInTool } from '$lib/enums'; -import type { AgenticSection } from '$lib/types'; +import type { AgenticSection, WriteFileMeta, WriteFileTitleMeta } from '$lib/types'; import { getFileTypeByExtension, tryParseToolResultObject } from '$lib/utils'; -export type WriteFileMeta = { - fileName: string; - filePath: string; - language: string; - content: string; - bytesWritten?: number; - resultMessage?: string; - errorMessage?: string; -}; - export function parseWriteFileMeta(section: AgenticSection): WriteFileMeta | null { const args = parseToolArgs(BuiltInTool.SERVER_WRITE_FILE, section, { partial: true }); @@ -51,3 +41,43 @@ export function parseWriteFileMeta(section: AgenticSection): WriteFileMeta | nul resultMessage }; } + +/** + * Title-tier meta for write_file blocks: everything the header and status + * pill render, obtained without parsing the embedded file content. The path + * comes from a targeted key extraction; the full parse runs only as a + * fallback for arg shapes the extraction can't see. + */ +export function parseWriteFileTitleMeta(section: AgenticSection): WriteFileTitleMeta | null { + if (section.toolName !== BuiltInTool.SERVER_WRITE_FILE || !section.toolArgs) return null; + + let rawPath: string | undefined = extractToolArgString(section.toolArgs, TOOL_ARG_PATH_KEYS); + + if (!rawPath) { + const args = parseToolArgs(BuiltInTool.SERVER_WRITE_FILE, section, { partial: true }); + const fallbackPath = args?.path ?? args?.file_path ?? args?.filePath; + + if (typeof fallbackPath === 'string' && fallbackPath) rawPath = fallbackPath; + } + + if (!rawPath) return null; + + const fileName = rawPath.split(FILE_PATH_SEPARATOR_REGEX).pop() || rawPath; + const language = + getFileTypeByExtension(rawPath)?.replace(CODE_BLOCK.TEXT_LANGUAGE_PREFIX_REGEX, '') ?? + CODE_BLOCK.DEFAULT_LANGUAGE; + const resultObj = tryParseToolResultObject(section.toolResult); + const bytesWritten = + resultObj && Number.isFinite(Number(resultObj.bytes)) ? Number(resultObj.bytes) : undefined; + const resultMessage = typeof resultObj?.result === 'string' ? resultObj.result : undefined; + const errorMessage = typeof resultObj?.error === 'string' ? resultObj.error : undefined; + + return { + bytesWritten, + errorMessage, + fileName, + filePath: rawPath, + language, + resultMessage + }; +} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserBubble.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserBubble.svelte index 65818c64bd42..569737ac3b7b 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserBubble.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserBubble.svelte @@ -54,7 +54,12 @@ {#if attachments && attachments.length > 0} <div class="mb-2 max-w-[80%]"> - <ChatAttachmentsList {attachments} imageHeight="h-40" readonly /> + <ChatAttachmentsList + {attachments} + imageHeight="max-h-40" + imageWidth="w-auto max-w-full" + readonly + /> </div> {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte index 5137e261f83c..ea9428e0712a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte @@ -46,49 +46,44 @@ isLastAssistantMessage ? !!agenticStore.getLastError(message.convId) : false ); - let permissionDismissed = $state(false); - const pendingPermission = $derived( isStreaming && isLastAssistantMessage ? agenticStore.getPendingPermissionRequest(message.convId) : null ); - let prevPendingRef: typeof pendingPermission = null; - $effect(() => { - if (pendingPermission !== prevPendingRef) { - prevPendingRef = pendingPermission; + // dismissal applies to the request object, so the next request ( new + // identity ) shows the card again without any reset bookkeeping + let dismissedPermission: typeof pendingPermission = $state(null); - if (pendingPermission) { - permissionDismissed = false; - } - } - }); + const visiblePermission = $derived( + pendingPermission && dismissedPermission !== pendingPermission ? pendingPermission : null + ); function handlePermission(decision: ToolPermissionDecision) { - permissionDismissed = true; + dismissedPermission = pendingPermission; agenticStore.resolvePermission(message.convId, decision); } - let continueDismissed = $state(false); - const pendingContinue = $derived( isStreaming && isLastAssistantMessage ? agenticStore.getPendingContinueRequest(message.convId) : false ); - let prevContinueRef = false; - $effect(() => { - if (pendingContinue !== prevContinueRef) { - prevContinueRef = pendingContinue; + let continueDismissed = $state(false); - if (pendingContinue) { - continueDismissed = false; - } + // the continue request is a plain boolean, so there is no identity to + // compare against; clear the dismissal whenever no request is pending so + // the next one starts from a clean state + $effect(() => { + if (!pendingContinue) { + continueDismissed = false; } }); + const showContinue = $derived(Boolean(pendingContinue) && !continueDismissed); + function handleContinue(shouldContinue: boolean) { continueDismissed = true; agenticStore.resolveContinue(message.convId, shouldContinue); @@ -238,15 +233,15 @@ {/each} {/if} - {#if pendingPermission && !permissionDismissed} + {#if visiblePermission} <ChatMessageActionCardPermissionRequest onDecision={handlePermission} - serverLabel={pendingPermission.serverLabel} - toolName={pendingPermission.toolName} + serverLabel={visiblePermission.serverLabel} + toolName={visiblePermission.toolName} /> {/if} - {#if pendingContinue && !continueDismissed} + {#if showContinue} <ChatMessageActionCardContinueRequest onDecision={handleContinue} /> {/if} </div> diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte index 4750a9f7c14f..0078225c08bc 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte @@ -1,5 +1,6 @@ <script lang="ts"> - import { ChatMessage, ChatMessageUserPending } from '$lib/components/app'; + import LazyChatMessage from './LazyChatMessage.svelte'; + import { ChatMessageUserPending } from '$lib/components/app'; import { MessageRole } from '$lib/enums'; import { agenticStore, chatStore, conversationsStore, settingsStore } from '$lib/stores'; import type { ChatMessageActions } from '$lib/types'; @@ -51,8 +52,9 @@ newExtras?: DatabaseMessageExtra[] ) => { onUserAction?.(); + // in-place edit: the store already updated activeMessages and no + // branch is created, so sibling info stays valid without a refetch await chatStore.editUserMessagePreserveResponses(message.id, newContent, newExtras); - refreshAllMessages(); }, editWithBranching: async ( @@ -72,7 +74,10 @@ ) => { onUserAction?.(); await chatStore.editAssistantMessage(message.id, newContent, shouldBranch); - refreshAllMessages(); + + // only a branch changes sibling info; an in-place edit already + // landed in activeMessages + if (shouldBranch) refreshAllMessages(); }, forkConversation: async ( @@ -97,9 +102,17 @@ const conversation = conversationsStore.activeConversation; if (conversation) { - conversationsStore.getConversationMessages(conversation.id).then((messages) => { - allConversationMessages = messages; - }); + // reuse the array loadConversation just read, when present; branch + // actions fall through to a fresh fetch + const preloaded = conversationsStore.consumeLastLoadedMessages(conversation.id); + + if (preloaded) { + allConversationMessages = preloaded; + } else { + conversationsStore.getConversationMessages(conversation.id).then((messages) => { + allConversationMessages = messages; + }); + } } else { allConversationMessages = []; } @@ -224,48 +237,76 @@ }); </script> -<div> - {#each displayMessages as { isLastAssistantMessage, isLastUserMessage, message, nextAssistantMessage, siblingInfo, toolMessages } (message.id)} - <ChatMessage - {chatActions} - class="mx-auto mt-12 w-full max-w-3xl" - {isLastAssistantMessage} - {isLastUserMessage} - {message} - {nextAssistantMessage} - {siblingInfo} - {toolMessages} - /> - {/each} - - {#if conversationsStore.activeConversation && agenticStore.getPendingSteeringMessageContent(conversationsStore.activeConversation!.id)} - {@const convId = conversationsStore.activeConversation!.id} - {@const pendingContent = agenticStore.getPendingSteeringMessageContent(convId)} - - {#if pendingContent} - <ChatMessageUserPending - class="mx-auto mt-12 w-full max-w-[48rem]" - content={pendingContent} - extras={agenticStore.getPendingSteeringMessageExtras(convId)} - onDelete={() => agenticStore.clearSteeringMessage(convId)} - onEdit={(newContent, extras) => - agenticStore.injectSteeringMessage(convId, newContent, extras)} - onSendImmediately={() => chatStore.abortCurrentFlow(convId)} - /> - {/if} - {:else if conversationsStore.activeConversation && chatStore.getPendingMessageContent(conversationsStore.activeConversation!.id)} - {@const convId = conversationsStore.activeConversation!.id} - {@const pendingContent = chatStore.getPendingMessageContent(convId)} - - {#if pendingContent} - <ChatMessageUserPending - class="mx-auto mt-12 w-full max-w-[48rem]" - content={pendingContent} - extras={chatStore.getPendingMessageExtras(convId)} - onDelete={() => chatStore.clearPendingMessage(convId)} - onEdit={(newContent, extras) => chatStore.injectPendingMessage(convId, newContent, extras)} - onSendImmediately={() => chatStore.abortCurrentFlow(convId)} +<!-- Re-created per conversation, so the CSS fade-in below plays on every + navigation into a chat route. --> +{#key conversationsStore.activeConversation?.id ?? 'new'} + <div class="chat-messages"> + {#each displayMessages as { isLastAssistantMessage, isLastUserMessage, message, nextAssistantMessage, siblingInfo, toolMessages } (message.id)} + <LazyChatMessage + {chatActions} + class="mx-auto mt-12 w-full max-w-3xl" + {isLastAssistantMessage} + {isLastUserMessage} + {message} + {nextAssistantMessage} + {siblingInfo} + {toolMessages} /> + {/each} + + {#if conversationsStore.activeConversation && agenticStore.getPendingSteeringMessageContent(conversationsStore.activeConversation!.id)} + {@const convId = conversationsStore.activeConversation!.id} + {@const pendingContent = agenticStore.getPendingSteeringMessageContent(convId)} + + {#if pendingContent} + <ChatMessageUserPending + class="mx-auto mt-12 w-full max-w-[48rem]" + content={pendingContent} + extras={agenticStore.getPendingSteeringMessageExtras(convId)} + onDelete={() => agenticStore.clearSteeringMessage(convId)} + onEdit={(newContent, extras) => + agenticStore.injectSteeringMessage(convId, newContent, extras)} + onSendImmediately={() => chatStore.abortCurrentFlow(convId)} + /> + {/if} + {:else if conversationsStore.activeConversation && chatStore.getPendingMessageContent(conversationsStore.activeConversation!.id)} + {@const convId = conversationsStore.activeConversation!.id} + {@const pendingContent = chatStore.getPendingMessageContent(convId)} + + {#if pendingContent} + <ChatMessageUserPending + class="mx-auto mt-12 w-full max-w-[48rem]" + content={pendingContent} + extras={chatStore.getPendingMessageExtras(convId)} + onDelete={() => chatStore.clearPendingMessage(convId)} + onEdit={(newContent, extras) => + chatStore.injectPendingMessage(convId, newContent, extras)} + onSendImmediately={() => chatStore.abortCurrentFlow(convId)} + /> + {/if} {/if} - {/if} -</div> + </div> +{/key} + +<style> + /* Compositor-friendly opacity fade; the keyed block re-creates the list per + * conversation, so the animation plays on every navigation into a chat. */ + .chat-messages { + animation: chat-messages-fade-in 150ms ease-out; + } + + @keyframes chat-messages-fade-in { + from { + opacity: 0; + } + to { + opacity: 1; + } + } + + @media (prefers-reduced-motion: reduce) { + .chat-messages { + animation: none; + } + } +</style> diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/LazyChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/LazyChatMessage.svelte new file mode 100644 index 000000000000..f9667bbbbef2 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/LazyChatMessage.svelte @@ -0,0 +1,105 @@ +<script lang="ts"> + import ChatMessage from './ChatMessage/ChatMessage.svelte'; + import { chatStore } from '$lib/stores'; + import type { ChatMessageActions } from '$lib/types'; + + interface Props { + chatActions: ChatMessageActions; + class?: string; + isLastAssistantMessage?: boolean; + isLastUserMessage?: boolean; + message: DatabaseMessage; + nextAssistantMessage?: DatabaseMessage | null; + siblingInfo?: ChatMessageSiblingInfo | null; + toolMessages?: DatabaseMessage[]; + } + + let { + chatActions, + class: className = '', + isLastAssistantMessage = false, + isLastUserMessage = false, + message, + nextAssistantMessage = null, + siblingInfo = null, + toolMessages = [] + }: Props = $props(); + + // A mounted message row is a whole component tree (contexts, effects, + // collapsibles, markdown blocks), and the cycle collector, GC and layout + // invalidation keep walking every live object and DOM node, even for + // rows the user never scrolls to. Mount the real tree only when the row + // approaches the viewport; until then the row is an empty placeholder + // that reserves its size through content-visibility. + let mounted = $state(false); + let wrapperEl: HTMLDivElement | undefined = $state(); + + $effect(() => { + if (mounted || !wrapperEl) return; + + const observer = new IntersectionObserver( + (entries) => { + if (entries.some((entry) => entry.isIntersecting)) { + mounted = true; + observer.disconnect(); + } + }, + // pre-mount a couple of viewport heights ahead of the scroll + // position so a fast scroll never meets an empty row + { rootMargin: '200% 0px' } + ); + + observer.observe(wrapperEl); + + return () => observer.disconnect(); + }); + + // Flows that target a row by id (pending edit) expect the message + // component and its effects to exist; mount the target row first + $effect(() => { + if (chatStore.pendingEditMessageId === message.id) { + mounted = true; + } + }); +</script> + +<div + bind:this={wrapperEl} + class:chat-message--synthetic={Boolean(message.isSynthetic)} + class="chat-message" +> + {#if mounted} + <ChatMessage + {chatActions} + class={className} + {isLastAssistantMessage} + {isLastUserMessage} + {message} + {nextAssistantMessage} + {siblingInfo} + {toolMessages} + /> + {/if} +</div> + +<style> + /* + * The browser skips layout and paint for messages outside the + * viewport. contain-intrinsic-size reuses the last rendered size + * once known; 500px sizes messages that have never been rendered. + */ + .chat-message { + --chat-message-intrinsic-size: 500px; + content-visibility: auto; + contain-intrinsic-size: auto var(--chat-message-intrinsic-size); + } + + /* + * Synthetic rows (e.g. the working-directory change) are small, so an + * accurate placeholder keeps the injected row from inflating the + * auto-scroll offset; the 500px default is for ordinary bubbles. + */ + .chat-message--synthetic { + --chat-message-intrinsic-size: 40px; + } +</style> diff --git a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte index 3ad3f24685f7..6cea95d0d09a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte @@ -315,13 +315,18 @@ <div style:padding-top={!isEmpty ? 'var(--chat-form-padding-top)' : undefined} class={[ - 'pointer-events-none md:sticky fixed mt-auto transition-all duration-200', + // animate the centered->bottomed move with transform, not bottom: + // layout-property transitions need the main thread every frame and + // stutter while a long conversation loads; transform transitions + // run on the compositor and stay smooth + 'pointer-events-none md:sticky fixed mt-auto transition-transform duration-200', deviceStore.isStandalone ? 'bottom-6 right-4 left-4' : deviceStore.isIOSSafari ? 'bottom-1 left-2 right-2' : 'bottom-2 right-2 left-2', - isEmpty ? 'md:bottom-[calc(50dvh-7rem)] 2xl:bottom-[calc(50dvh-4rem)]' : 'md:bottom-4' + 'md:bottom-4', + isEmpty ? 'md:translate-y-[calc(-50dvh+8rem)] 2xl:translate-y-[calc(-50dvh+5rem)]' : '' ]} > <ChatScreenGreeting {isEmpty} /> diff --git a/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte b/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte index c54b981cde8e..ecdd75bda50b 100644 --- a/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte +++ b/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte @@ -65,7 +65,12 @@ <IconComponent class={cn('shrink-0 text-muted-foreground/60 mt-0.75', iconClass)} /> {/if} - <span class={cn('text-sm font-medium', shimmerTitle ? 'shimmer-text' : 'text-foreground/80')}> + <span + class={cn( + 'min-w-0 overflow-x-auto text-sm font-medium', + shimmerTitle ? 'shimmer-text' : 'text-foreground/80' + )} + > {#if titleSnippet} {@render titleSnippet()} {:else} diff --git a/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte b/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte index 0ad6ea61fc9d..610923b9cfe2 100644 --- a/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte +++ b/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte @@ -66,7 +66,12 @@ <IconComponent class={cn('shrink-0 text-muted-foreground/60 mt-0.5', iconClass)} /> {/if} - <span class={cn('text-sm font-medium', shimmerTitle ? 'shimmer-text' : 'text-foreground/80')}> + <span + class={cn( + 'min-w-0 overflow-x-auto text-sm font-medium', + shimmerTitle ? 'shimmer-text' : 'text-foreground/80' + )} + > {#if titleSnippet} {@render titleSnippet()} {:else} diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte index 87b41bd00dee..c217a769a6b4 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte @@ -1,23 +1,12 @@ <script lang="ts"> import '$lib/styles/katex-custom.scss'; + import { getMarkdownProcessor, type MarkdownProcessor } from './markdown-processor'; import { getCodeInfoFromTarget, getHastNodeId, getMdastNodeHash, isAppendMode } from './markdown-utils'; - import { rehypeEnhanceCodeBlocks } from './plugins/rehype/enhance-code-blocks'; - import { rehypeEnhanceLinks } from './plugins/rehype/enhance-links'; - import { rehypeEnhanceMermaidBlocks } from './plugins/rehype/enhance-mermaid-blocks'; - import { rehypeEnhanceSvgBlocks } from './plugins/rehype/enhance-svg-blocks'; - import { rehypeFileBadge } from './plugins/rehype/file-badge'; - import { rehypeMermaidPre } from './plugins/rehype/mermaid-pre'; - import { rehypeRtlSupport } from './plugins/rehype/rehype-rtl-support'; - import { rehypeResolveAttachmentImages } from './plugins/rehype/resolve-attachment-images'; - import { rehypeSvgPre } from './plugins/rehype/svg-pre'; - import { rehypeRestoreTableHtml } from './plugins/rehype/table-html-restorer'; - import { remarkLiteralHtml } from './plugins/remark/literal-html'; - import { browser } from '$app/environment'; import { ActionIconCopyToClipboard, CodeBlockActions, @@ -38,10 +27,10 @@ MERMAID_WRAPPER_CLASS, SETTINGS_KEYS, SVG, - TOGGLE_SOURCE_BTN_CLASS + TOGGLE_SOURCE_BTN_CLASS, + UI_DATA_ATTRS } from '$lib/constants'; import { BooleanString, ColorMode, UrlProtocol } from '$lib/enums'; - import { FileTypeText } from '$lib/enums/files.enums'; import { createAutoScrollController } from '$lib/hooks/use-auto-scroll.svelte'; import { settingsStore } from '$lib/stores'; import type { DatabaseMessageExtra } from '$lib/types/database'; @@ -58,17 +47,8 @@ import type { Root as HastRoot, RootContent as HastRootContent } from 'hast'; import githubLightCss from 'highlight.js/styles/github.css?inline'; import githubDarkCss from 'highlight.js/styles/github-dark.css?inline'; - import { all as lowlightAll } from 'lowlight'; import type { Root as MdastRoot } from 'mdast'; import { mode } from 'mode-watcher'; - import rehypeHighlight from 'rehype-highlight'; - import rehypeKatex from 'rehype-katex'; - import rehypeStringify from 'rehype-stringify'; - import { remark } from 'remark'; - import remarkBreaks from 'remark-breaks'; - import remarkGfm from 'remark-gfm'; - import remarkMath from 'remark-math'; - import remarkRehype from 'remark-rehype'; import { onDestroy, tick } from 'svelte'; import { SvelteMap } from 'svelte/reactivity'; @@ -144,44 +124,6 @@ const transformCache = new SvelteMap<string, string>(); let previousContent = ''; - const themeStyleId = `highlight-theme-${(window.idxThemeStyle = (window.idxThemeStyle ?? 0) + 1)}`; - - let processor = $derived(() => { - void attachments; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let proc: any = remark().use(remarkGfm); // GitHub Flavored Markdown - - if (!disableMath) { - proc = proc.use(remarkMath); // Parse $inline$ and $$block$$ math - } - - proc = proc - .use(remarkBreaks) // Convert line breaks to <br> - .use(remarkLiteralHtml) // Treat raw HTML as literal text with preserved indentation - .use(remarkRehype); // Convert Markdown AST to rehype - - if (!disableMath) { - proc = proc.use(rehypeKatex); // Render math using KaTeX - } - - return proc - .use(rehypeHighlight, { - aliases: { [FileTypeText.XML]: [FileTypeText.SVELTE, FileTypeText.VUE] }, - languages: lowlightAll - }) // Add syntax highlighting - .use(rehypeRestoreTableHtml) // Restore limited HTML (e.g., <br>, <ul>) inside Markdown tables - .use(rehypeEnhanceLinks) // Add target="_blank" to links - .use(rehypeFileBadge) // Render file:// anchors as inline badge chips - .use(rehypeMermaidPre) // Convert mermaid blocks to <pre class="mermaid"> - .use(rehypeSvgPre) // Convert svg blocks to <pre class="svg-block"> - .use(rehypeEnhanceCodeBlocks) // Wrap code blocks with header and actions - .use(rehypeEnhanceMermaidBlocks) // Wrap mermaid blocks with header and actions - .use(rehypeEnhanceSvgBlocks) // Wrap svg blocks with header and actions - .use(rehypeResolveAttachmentImages, { attachments }) - .use(rehypeRtlSupport) // Add bidirectional text support - .use(rehypeStringify, { allowDangerousHtml: true }); // Convert to HTML string - }); - /** * Removes click event listeners from copy and preview buttons. * Called on component destroy. @@ -201,33 +143,22 @@ } } - /** - * Removes this component's highlight.js theme style from the document head. - * Called on component destroy to clean up injected styles. - */ - function cleanupHighlightTheme() { - if (!browser) return; - - const existingTheme = document.getElementById(themeStyleId); - - existingTheme?.remove(); - } - /** * Loads the appropriate highlight.js theme based on dark/light mode. - * Injects a scoped style element into the document head. + * One shared style element for every markdown block, mirroring + * SyntaxHighlightedCode.svelte. The old per-instance copies duplicated the + * full theme CSS once per rendered message, which grows without bound in + * long conversations. * @param isDark - Whether to load the dark theme (true) or light theme (false) */ function loadHighlightTheme(isDark: boolean) { - if (!browser) return; - - const existingTheme = document.getElementById(themeStyleId); - - existingTheme?.remove(); + document + .querySelectorAll(`style[${UI_DATA_ATTRS.HIGHLIGHT_THEME_PREVIEW}]`) + .forEach((style) => style.remove()); const style = document.createElement('style'); - style.id = themeStyleId; + style.setAttribute(UI_DATA_ATTRS.HIGHLIGHT_THEME_PREVIEW, BooleanString.TRUE); style.textContent = isDark ? githubDarkCss : githubLightCss; document.head.appendChild(style); @@ -247,7 +178,7 @@ * @returns Object containing the HTML string and cache hash */ async function transformMdastNode( - processorInstance: ReturnType<typeof processor>, + processorInstance: MarkdownProcessor, node: unknown, index: number ): Promise<{ html: string; hash: string }> { @@ -369,7 +300,7 @@ if (prefixMarkdown.trim()) { const normalizedPrefix = preprocessLaTeX(prefixMarkdown); - const processorInstance = processor(); + const processorInstance = getMarkdownProcessor({ attachments, disableMath }); const ast = processorInstance.parse(normalizedPrefix) as MdastRoot; const mdastChildren = (ast as { children?: unknown[] }).children ?? []; const nextBlocks: MarkdownBlock[] = []; @@ -419,7 +350,7 @@ incompleteCodeBlock = null; const normalized = preprocessLaTeX(markdown); - const processorInstance = processor(); + const processorInstance = getMarkdownProcessor({ attachments, disableMath }); const ast = processorInstance.parse(normalized) as MdastRoot; const mdastChildren = (ast as { children?: unknown[] }).children ?? []; const stableCount = Math.max(mdastChildren.length - 1, 0); @@ -858,7 +789,6 @@ onDestroy(() => { cleanupEventListeners(); - cleanupHighlightTheme(); streamingAutoScroll.destroy(); }); </script> diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-content.css b/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-content.css index cada489ca97a..b0ca884ae0ff 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-content.css +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-content.css @@ -1,3 +1,10 @@ +/* Long unbreakable content (inline code, paths, hashes) wraps inside the + column; break-word keeps the min-content width intact, so wide tables and + code blocks still scroll in their own containers. */ +.markdown-content { + overflow-wrap: break-word; +} + .markdown-block--unstable { display: contents; } @@ -429,15 +436,19 @@ div.markdown-user-content :global(.table-wrapper) { /* Enhanced images */ .markdown-content :global(img) { - transition: all 0.3s ease; + transition: + transform 200ms ease-out, + box-shadow 200ms ease-out; cursor: pointer; } -.markdown-content :global(img:hover) { - transform: scale(1.02); - box-shadow: - 0 10px 15px -3px rgb(0 0 0 / 0.1), - 0 4px 6px -4px rgb(0 0 0 / 0.1); +@media (hover: hover) and (pointer: fine) { + .markdown-content :global(img:hover) { + transform: scale(1.02); + box-shadow: + 0 10px 15px -3px rgb(0 0 0 / 0.1), + 0 4px 6px -4px rgb(0 0 0 / 0.1); + } } /* Image zoom overlay */ diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts b/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts new file mode 100644 index 000000000000..57ded3e99ada --- /dev/null +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts @@ -0,0 +1,114 @@ +// Shared remark/rehype pipeline factory for MarkdownContent. +// +// The frozen plugin chain is expensive to build ( ~15 plugin instances ), +// and MarkdownContent used to rebuild it on every processMarkdown call: +// once per block at mount, and again on every coalesced chunk while +// streaming. Pipelines without attachments are shared process-wide per +// math flag; attachment-bearing pipelines are cached by the attachments +// array identity, which changes whenever extras are updated. + +import { rehypeEnhanceCodeBlocks } from './plugins/rehype/enhance-code-blocks'; +import { rehypeEnhanceLinks } from './plugins/rehype/enhance-links'; +import { rehypeEnhanceMermaidBlocks } from './plugins/rehype/enhance-mermaid-blocks'; +import { rehypeEnhanceSvgBlocks } from './plugins/rehype/enhance-svg-blocks'; +import { rehypeEnhanceTables } from './plugins/rehype/enhance-tables'; +import { rehypeFileBadge } from './plugins/rehype/file-badge'; +import { rehypeMermaidPre } from './plugins/rehype/mermaid-pre'; +import { rehypeRtlSupport } from './plugins/rehype/rehype-rtl-support'; +import { rehypeResolveAttachmentImages } from './plugins/rehype/resolve-attachment-images'; +import { rehypeSvgPre } from './plugins/rehype/svg-pre'; +import { rehypeRestoreTableHtml } from './plugins/rehype/table-html-restorer'; +import { remarkLiteralHtml } from './plugins/remark/literal-html'; +import { FileTypeText } from '$lib/enums/files.enums'; +import type { DatabaseMessageExtra } from '$lib/types/database'; +import type { Root as HastRoot } from 'hast'; +import { all as lowlightAll } from 'lowlight'; +import type { Root as MdastRoot } from 'mdast'; +import rehypeHighlight from 'rehype-highlight'; +import rehypeKatex from 'rehype-katex'; +import rehypeStringify from 'rehype-stringify'; +import { remark } from 'remark'; +import remarkBreaks from 'remark-breaks'; +import remarkGfm from 'remark-gfm'; +import remarkMath from 'remark-math'; +import remarkRehype from 'remark-rehype'; + +export interface MarkdownProcessor { + parse(markdown: string): MdastRoot; + run(tree: MdastRoot): Promise<HastRoot>; + stringify(tree: HastRoot): string; +} + +export interface MarkdownProcessorOptions { + attachments?: DatabaseMessageExtra[]; + disableMath?: boolean; +} + +const sharedPipelines = new Map<string, MarkdownProcessor>(); +const attachmentPipelines = new WeakMap<object, MarkdownProcessor>(); + +function buildPipeline({ + attachments, + disableMath = false +}: MarkdownProcessorOptions): MarkdownProcessor { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let proc: any = remark().use(remarkGfm); // GitHub Flavored Markdown + + if (!disableMath) { + proc = proc.use(remarkMath); // Parse $inline$ and $$block$$ math + } + + proc = proc + .use(remarkBreaks) // Convert line breaks to <br> + // Treat raw HTML as literal text with preserved indentation + .use(remarkLiteralHtml) + .use(remarkRehype); // Convert Markdown AST to rehype + + if (!disableMath) { + proc = proc.use(rehypeKatex); // Render math using KaTeX + } + + const pipeline = proc + .use(rehypeHighlight, { + aliases: { [FileTypeText.XML]: [FileTypeText.SVELTE, FileTypeText.VUE] }, + languages: lowlightAll + }) // Add syntax highlighting + .use(rehypeRestoreTableHtml) // Restore limited HTML (e.g. <br>, <ul>) inside Markdown tables + .use(rehypeEnhanceTables) // Wrap tables in a horizontal scroll container + .use(rehypeEnhanceLinks) // Add target="_blank" to links + .use(rehypeFileBadge) // Render file:// anchors as inline badge chips + .use(rehypeMermaidPre) // Convert mermaid blocks to <pre class="mermaid"> + .use(rehypeSvgPre) // Convert svg blocks to <pre class="svg-block"> + .use(rehypeEnhanceCodeBlocks) // Wrap code blocks with header and actions + .use(rehypeEnhanceMermaidBlocks) // Wrap mermaid blocks with header and actions + .use(rehypeEnhanceSvgBlocks) // Wrap svg blocks with header and actions + .use(rehypeResolveAttachmentImages, { attachments }) + .use(rehypeRtlSupport) // Add bidirectional text support + .use(rehypeStringify, { allowDangerousHtml: true }); // Convert to HTML string + + return pipeline as MarkdownProcessor; +} + +export function getMarkdownProcessor(options: MarkdownProcessorOptions): MarkdownProcessor { + if (options.attachments && options.attachments.length > 0) { + let cached = attachmentPipelines.get(options.attachments); + + if (!cached) { + cached = buildPipeline(options); + attachmentPipelines.set(options.attachments, cached); + } + + return cached; + } + + const key = String(Boolean(options.disableMath)); + + let cached = sharedPipelines.get(key); + + if (!cached) { + cached = buildPipeline(options); + sharedPipelines.set(key, cached); + } + + return cached; +} diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/enhance-tables.ts b/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/enhance-tables.ts new file mode 100644 index 000000000000..b08bedba07d6 --- /dev/null +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/enhance-tables.ts @@ -0,0 +1,34 @@ +/** + * Rehype plugin to wrap tables in a horizontal scroll container. + * + * A bare <table> keeps its content-driven minimum width, which propagates up + * the layout and can stretch the chat column past the window. Wrapping in + * div.table-wrapper makes the wrapper the scroll container (styled in + * markdown-content.css), so wide tables scroll in place instead. + */ + +import type { Element, ElementContent, Root } from 'hast'; +import type { Plugin } from 'unified'; +import { visit } from 'unist-util-visit'; + +export const rehypeEnhanceTables: Plugin<[], Root> = () => { + return (tree: Root) => { + visit(tree, 'element', (node: Element, index, parent) => { + if (node.tagName !== 'table' || !parent || index === undefined) return; + + // already wrapped (e.g. nested tables in raw HTML input) + const parentClass = parent.type === 'element' ? parent.properties?.className : undefined; + + if (Array.isArray(parentClass) && parentClass.includes('table-wrapper')) return; + + const wrapper: Element = { + children: [node as ElementContent], + properties: { className: ['table-wrapper'] }, + tagName: 'div', + type: 'element' + }; + + parent.children[index] = wrapper; + }); + }; +}; diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts index e3241373e8b4..d93ae642937a 100644 --- a/tools/ui/src/lib/constants/index.ts +++ b/tools/ui/src/lib/constants/index.ts @@ -16,6 +16,7 @@ export * from './context-gauge-popup.constants'; export * from './conversation-import.constants'; export * from './binary-detection.constants'; export * from './content-detection.constants'; +export * from './tool-call-args.constants'; export * from './tool-ui.constants'; export * from './cache.constants'; export * from './chat-form.constants'; diff --git a/tools/ui/src/lib/constants/supported-file-types.constants.ts b/tools/ui/src/lib/constants/supported-file-types.constants.ts index a6bcefaa157e..5675d682fd77 100644 --- a/tools/ui/src/lib/constants/supported-file-types.constants.ts +++ b/tools/ui/src/lib/constants/supported-file-types.constants.ts @@ -40,6 +40,10 @@ export const VIDEO_FILE_TYPES = { [FileTypeVideo.OGG]: { extensions: [FileExtensionVideo.OGG], mimeTypes: [MimeTypeVideo.OGG] + }, + [FileTypeVideo.WEBM]: { + extensions: [FileExtensionVideo.WEBM], + mimeTypes: [MimeTypeVideo.WEBM] } } as const; diff --git a/tools/ui/src/lib/constants/tool-call-args.constants.ts b/tools/ui/src/lib/constants/tool-call-args.constants.ts new file mode 100644 index 000000000000..e74260be2993 --- /dev/null +++ b/tools/ui/src/lib/constants/tool-call-args.constants.ts @@ -0,0 +1,23 @@ +// Tool-args and tool-result parsing helpers: the file tools' path field +// aliases, the JSON container gates for result blobs, and the targeted +// string-field pattern used for cheap title-tier extraction. + +/** + * Field aliases the file tools accept for the path argument. Tool contracts + * drifted over time: some models emit `file_path` / `filePath`. + */ +export const TOOL_ARG_PATH_KEYS: readonly string[] = ['path', 'file_path', 'filePath']; + +/** Opening character of a JSON object; only an object root can carry fields. */ +export const JSON_OBJECT_OPEN = '{'; + +/** Opening character of a JSON array; successful sandbox output is one. */ +export const JSON_ARRAY_OPEN = '['; + +/** + * Matches `"<key>": "<value>"` in a JSON args blob ( whitespace between + * tokens allowed ), capturing the raw string literal so only that literal + * gets decoded; escaped quotes stay inside the value group. `{key}` is + * replaced with the field name before use. + */ +export const TOOL_ARG_STRING_FIELD_PATTERN_TEMPLATE = '"{key}"\\s*:\\s*"((?:[^"\\\\]|\\\\.)*)"'; diff --git a/tools/ui/src/lib/enums/files.enums.ts b/tools/ui/src/lib/enums/files.enums.ts index 0185da4783e2..b8cde6061c8e 100644 --- a/tools/ui/src/lib/enums/files.enums.ts +++ b/tools/ui/src/lib/enums/files.enums.ts @@ -38,7 +38,8 @@ export enum FileTypeAudio { export enum FileTypeVideo { MP4 = 'mp4', - OGG = 'ogg' + OGG = 'ogg', + WEBM = 'webm' } export enum FileTypePdf { @@ -104,7 +105,8 @@ export enum FileExtensionAudio { export enum FileExtensionVideo { MP4 = '.mp4', - OGG = '.ogg' + OGG = '.ogg', + WEBM = '.webm' } export enum FileExtensionPdf { @@ -203,7 +205,8 @@ export enum MimeTypeAudio { export enum MimeTypeVideo { MP4 = 'video/mp4', - OGG = 'video/ogg' + OGG = 'video/ogg', + WEBM = 'video/webm' } export enum MimeTypeImage { diff --git a/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts b/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts index 21deed32d307..cb361aad8a4e 100644 --- a/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts +++ b/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts @@ -47,7 +47,7 @@ export function useToolsPanel(): UseToolsPanelReturn { if (toolsStore.toolGroups.length > 0) return null; - // Tools endpoint is unreachable (404) — server started without --tools + // Tools endpoint unreachable (403) — server started without tools if (toolsStore.isToolsEndpointUnreachable) { return `To enable Server Tools you need to run llama-server with ${CLI_FLAGS.TOOLS} all or ${CLI_FLAGS.TOOLS} <name> flag. To see MCP Tools you need to add / enable MCP Server(s).`; } diff --git a/tools/ui/src/lib/stores/agentic/index.svelte.ts b/tools/ui/src/lib/stores/agentic/index.svelte.ts index 50db1be0cc30..121ee7739cdb 100644 --- a/tools/ui/src/lib/stores/agentic/index.svelte.ts +++ b/tools/ui/src/lib/stores/agentic/index.svelte.ts @@ -315,8 +315,14 @@ class AgenticStore { // Clear any pending permissions/continue requests for this conversation when starting a new flow this.gates.clear(conversationId); - // Ensure server tools are fetched before checking if agentic is enabled - if (toolsStore.serverTools.length === 0 && !toolsStore.loading) { + // Ensure server tools are fetched before checking if agentic is enabled. + // A disabled /tools endpoint stays disabled for the life of the server, + // so the tools panel is the only place that probes it again. + if ( + toolsStore.serverTools.length === 0 && + !toolsStore.loading && + !toolsStore.isToolsEndpointUnreachable + ) { await toolsStore.fetchServerTools(); } diff --git a/tools/ui/src/lib/stores/chat/index.svelte.ts b/tools/ui/src/lib/stores/chat/index.svelte.ts index 296c2cca5829..4bdcc6845668 100644 --- a/tools/ui/src/lib/stores/chat/index.svelte.ts +++ b/tools/ui/src/lib/stores/chat/index.svelte.ts @@ -55,7 +55,6 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { string, { response: string; messageId: string; model?: string | null } >(); - currentResponse = $state(''); errorDialogState = $state<ErrorDialogState | null>(null); // true while the active conversation has a local pipe (send, attach or resume-wait) isLoading = $derived(this.activity.isLocal(conversationsStore.activeConversation?.id ?? '')); @@ -256,8 +255,6 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { } this.chatStreamingStates.delete(convId); - - if (convId === conversationsStore.activeConversation?.id) this.currentResponse = ''; } clearEditMode(): void { this.isEditModeActive = false; @@ -272,11 +269,6 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { this.pendingMessages.delete(convId); } - /** Reset per-view state when (re)mounting the empty chat screen. */ - clearUIState(): void { - this.currentResponse = ''; - } - consumePendingDraft(): { message: string; files: ChatUploadedFile[] } | null { if (!this.pendingDraftMessage && this.pendingDraftFiles.length === 0) return null; @@ -766,8 +758,6 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { model: model ?? this.chatStreamingStates.get(convId)?.model, response }); - - if (convId === conversationsStore.activeConversation?.id) this.currentResponse = response; } setEditModeActive(handler: (files: File[]) => void): void { @@ -1244,7 +1234,6 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { syncLoadingStateForChat(convId: string): void { const s = this.chatStreamingStates.get(convId); - this.currentResponse = s?.response || ''; this.processing.setActiveConversation(convId); // Sync streaming content to activeMessages so UI displays current content diff --git a/tools/ui/src/lib/stores/conversations/index.svelte.ts b/tools/ui/src/lib/stores/conversations/index.svelte.ts index df5b1ecef317..c4fea2e4eee1 100644 --- a/tools/ui/src/lib/stores/conversations/index.svelte.ts +++ b/tools/ui/src/lib/stores/conversations/index.svelte.ts @@ -52,6 +52,13 @@ class ConversationsStore implements ConversationsPreferencesHost { /** In-flight init run; shared by concurrent callers, reset on failure to allow retry */ private initPromise: Promise<void> | null = null; + /** + * Messages loadConversation just read, handed off once so the chat + * screen can reuse them for sibling info instead of re-fetching the + * whole conversation a second time. + */ + private lastLoadedMessages: { convId: string; messages: DatabaseMessage[] } | null = null; + /** * Memo of the last findMessageIndex() lookup. Streaming calls it once per * chunk for the same message, so a validated cache hit keeps that O(1) @@ -88,7 +95,13 @@ class ConversationsStore implements ConversationsPreferencesHost { } if (this.activeConversation?.id === id) { - this.activeConversation = { ...this.activeConversation, ...updates }; + // field-wise, not object replacement: effects that track the active + // conversation identity would otherwise refire on every rename or pin + const target = this.activeConversation as unknown as Record<string, unknown>; + + for (const [key, value] of Object.entries(updates)) { + if (target[key] !== value) target[key] = value; + } } } @@ -202,11 +215,8 @@ class ConversationsStore implements ConversationsPreferencesHost { const updates = await DatabaseService.bulkToggleConversationPins(convIds); const activeId = this.activeConversation?.id; - if (activeId && updates.has(activeId)) { - this.activeConversation = { - ...this.activeConversation!, - pinned: updates.get(activeId)! - }; + if (this.activeConversation && activeId && updates.has(activeId)) { + this.activeConversation.pinned = updates.get(activeId)!; } for (let i = 0; i < this.conversations.length; i++) { @@ -236,6 +246,17 @@ class ConversationsStore implements ConversationsPreferencesHost { this.preferences.resetPending(); } + /** One-shot handoff of the messages the last loadConversation read. */ + consumeLastLoadedMessages(convId: string): DatabaseMessage[] | null { + if (this.lastLoadedMessages?.convId !== convId) return null; + + const messages = this.lastLoadedMessages.messages; + + this.lastLoadedMessages = null; + + return messages; + } + /** * Creates a new conversation and navigates to it * @param name - Optional name for the conversation @@ -509,22 +530,15 @@ class ConversationsStore implements ConversationsPreferencesHost { // it doesn't belong to this conversation. this.preferences.pendingCwd = null; - this.activeConversation = conversation; - - if (conversation.currNode) { - const allMessages = await DatabaseService.getConversationMessages(convId); - const filteredMessages = filterByLeafNodeId( - allMessages, - conversation.currNode, - false - ) as DatabaseMessage[]; + const allMessages = await DatabaseService.getConversationMessages(convId); - this.activeMessages = filteredMessages; - } else { - const messages = await DatabaseService.getConversationMessages(convId); - - this.activeMessages = messages; - } + // set conversation and messages in one sync block so effects never see + // the new conversation with the previous conversation's messages + this.lastLoadedMessages = { convId, messages: allMessages }; + this.activeConversation = conversation; + this.activeMessages = conversation.currNode + ? (filterByLeafNodeId(allMessages, conversation.currNode, false) as DatabaseMessage[]) + : allMessages; return true; } catch (error) { @@ -558,7 +572,7 @@ class ConversationsStore implements ConversationsPreferencesHost { const currentLeafNodeId = findLeafNode(allMessages, siblingId); await DatabaseService.updateCurrentNode(this.activeConversation.id, currentLeafNodeId); - this.activeConversation = { ...this.activeConversation, currNode: currentLeafNodeId }; + this.activeConversation.currNode = currentLeafNodeId; await this.refreshActiveMessages(); if (rootMessage && this.activeMessages.length > 0) { @@ -694,7 +708,7 @@ class ConversationsStore implements ConversationsPreferencesHost { } if (this.activeConversation?.id === targetId) { - this.activeConversation = { ...this.activeConversation, lastModified: now }; + this.activeConversation.lastModified = now; } DatabaseService.updateConversation(targetId, { lastModified: now }).catch((error) => @@ -710,7 +724,7 @@ class ConversationsStore implements ConversationsPreferencesHost { if (!this.activeConversation) return; await DatabaseService.updateCurrentNode(this.activeConversation.id, nodeId); - this.activeConversation = { ...this.activeConversation, currNode: nodeId }; + this.activeConversation.currNode = nodeId; } /** diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.ts index db05e3cd55b1..1d4133408b54 100644 --- a/tools/ui/src/lib/stores/tools.svelte.ts +++ b/tools/ui/src/lib/stores/tools.svelte.ts @@ -32,7 +32,7 @@ import { mcpStore } from '$lib/stores/mcp/index.svelte'; import { modelsStore } from '$lib/stores/models/index.svelte'; import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { OpenAIToolDefinition, ToolEntry, ToolGroup } from '$lib/types'; -import { buildSandboxToolDefinition } from '$lib/utils'; +import { ApiError, buildSandboxToolDefinition } from '$lib/utils'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; /** Stable selection identity for a tool, shared by the disabled set and the permission store */ @@ -246,13 +246,10 @@ class ToolsStore { toolInfos.filter((info) => info.uses_cwd).map((info) => info.tool) ); } catch (err) { - const errorMessage = err instanceof Error ? err.message : String(err); - - this._error = errorMessage; + this._error = err instanceof Error ? err.message : String(err); // 403 from /tools means the server was started without --tools - // TODO: check status code instead of relying on message - if (errorMessage.includes('this feature is disabled')) { + if (err instanceof ApiError && err.status === 403) { this._toolsEndpointUnreachable = true; console.info('[ToolsStore] Server tools are disabled on the server'); } else { diff --git a/tools/ui/src/lib/types/index.ts b/tools/ui/src/lib/types/index.ts index d91c2811a4a0..333c1bd3cd8f 100644 --- a/tools/ui/src/lib/types/index.ts +++ b/tools/ui/src/lib/types/index.ts @@ -209,7 +209,16 @@ export type { export type { DesktopIconStripItem } from './navigation'; // Tools types -export type { ToolEntry, ToolGroup, ToolUiEntry } from './tools'; +export type { + EditFileEdit, + EditFileMeta, + EditFileTitleMeta, + ToolEntry, + ToolGroup, + ToolUiEntry, + WriteFileMeta, + WriteFileTitleMeta +} from './tools'; // Reasoning export type { ReasoningEffortLevel } from './reasoning'; diff --git a/tools/ui/src/lib/types/tools.d.ts b/tools/ui/src/lib/types/tools.d.ts index edcec65c7037..fa8963bd1f8f 100644 --- a/tools/ui/src/lib/types/tools.d.ts +++ b/tools/ui/src/lib/types/tools.d.ts @@ -31,3 +31,50 @@ export interface ToolGroup { serverId?: string; tools: ToolEntry[]; } + +export interface WriteFileMeta { + fileName: string; + filePath: string; + language: string; + content: string; + bytesWritten?: number; + resultMessage?: string; + errorMessage?: string; +} + +/** Everything the write_file block title and status pill show; the full meta + * ( with the embedded file content ) stays body-only so collapsed blocks + * never parse the content blob. */ +export interface WriteFileTitleMeta { + fileName: string; + filePath: string; + language: string; + bytesWritten?: number; + resultMessage?: string; + errorMessage?: string; +} + +export interface EditFileEdit { + oldText: string; + newText: string; +} + +export interface EditFileMeta { + fileName: string; + filePath: string; + edits: EditFileEdit[]; + resultMessage?: string; + editsApplied?: number; + errorMessage?: string; +} + +/** Everything the edit_file block title and status pill show; the full meta + * ( with the embedded edit strings ) stays body-only so collapsed blocks + * never parse the args blob. */ +export interface EditFileTitleMeta { + fileName: string; + filePath: string; + resultMessage?: string; + editsApplied?: number; + errorMessage?: string; +} diff --git a/tools/ui/src/lib/utils/agentic.ts b/tools/ui/src/lib/utils/agentic.ts index cd150c5efbff..28b3f43ee048 100644 --- a/tools/ui/src/lib/utils/agentic.ts +++ b/tools/ui/src/lib/utils/agentic.ts @@ -109,6 +109,89 @@ function deriveSingleTurnSections( return sections; } +interface TurnSectionsCacheEntry { + content: string | undefined; + extra: DatabaseMessageExtra[] | undefined; + reasoningContent: string | undefined; + toolCalls: string | undefined; + toolMessageContents: (string | undefined)[]; + toolMessageExtras: (DatabaseMessageExtra[] | undefined)[]; + toolMessages: DatabaseMessage[]; + sections: AgenticSection[]; +} + +const turnSectionsCache = new WeakMap<DatabaseMessage, TurnSectionsCacheEntry>(); + +function isTurnCacheValid( + entry: TurnSectionsCacheEntry, + message: DatabaseMessage, + toolMessages: DatabaseMessage[] +): boolean { + if ( + entry.content !== message.content || + entry.reasoningContent !== message.reasoningContent || + entry.toolCalls !== message.toolCalls || + entry.extra !== message.extra + ) { + return false; + } + + if (entry.toolMessages.length !== toolMessages.length) return false; + + for (let i = 0; i < toolMessages.length; i++) { + if (entry.toolMessages[i] !== toolMessages[i]) return false; + + if (entry.toolMessageContents[i] !== toolMessages[i].content) return false; + + if (entry.toolMessageExtras[i] !== toolMessages[i].extra) return false; + } + + return true; +} + +/** + * deriveSingleTurnSections with structural reuse for completed turns. + * + * deriveAgenticSections runs in a $derived invalidated per streamed chunk, but + * only the last turn actually changes. Messages mutate in place and are never + * replaced, so a WeakMap keyed by the turn's assistant message plus reference + * checks on every field deriveSingleTurnSections reads detects any change. A + * cache hit also returns the same section objects, keeping downstream props + * stable so tool blocks skip their per-chunk re-derive. The streaming turn + * recomputes uncached on every chunk. + */ +function deriveTurnSections( + message: DatabaseMessage, + toolMessages: DatabaseMessage[], + streamingToolCalls: ApiChatCompletionToolCall[], + isStreaming: boolean +): AgenticSection[] { + if (isStreaming || streamingToolCalls.length > 0) { + return deriveSingleTurnSections(message, toolMessages, streamingToolCalls, isStreaming); + } + + const cached = turnSectionsCache.get(message); + + if (cached && isTurnCacheValid(cached, message, toolMessages)) { + return cached.sections; + } + + const sections = deriveSingleTurnSections(message, toolMessages, [], false); + + turnSectionsCache.set(message, { + content: message.content, + extra: message.extra, + reasoningContent: message.reasoningContent, + sections, + toolCalls: message.toolCalls, + toolMessageContents: toolMessages.map((tm) => tm.content), + toolMessageExtras: toolMessages.map((tm) => tm.extra), + toolMessages + }); + + return sections; +} + /** * Derives display sections from structured message data. * @@ -132,13 +215,13 @@ export function deriveAgenticSections( const hasAssistantContinuations = toolMessages.some((m) => m.role === MessageRole.ASSISTANT); if (!hasAssistantContinuations) { - return deriveSingleTurnSections(message, toolMessages, streamingToolCalls, isStreaming); + return deriveTurnSections(message, toolMessages, streamingToolCalls, isStreaming); } const sections: AgenticSection[] = []; const firstTurnToolMsgs = collectToolMessages(toolMessages, 0); - sections.push(...deriveSingleTurnSections(message, firstTurnToolMsgs)); + sections.push(...deriveTurnSections(message, firstTurnToolMsgs, [], false)); let i = firstTurnToolMsgs.length; @@ -150,7 +233,7 @@ export function deriveAgenticSections( const isLastTurn = i + 1 + turnToolMsgs.length >= toolMessages.length; sections.push( - ...deriveSingleTurnSections( + ...deriveTurnSections( msg, turnToolMsgs, isLastTurn ? streamingToolCalls : [], diff --git a/tools/ui/src/lib/utils/branching.ts b/tools/ui/src/lib/utils/branching.ts index 6c2c895cbe17..43d33d424393 100644 --- a/tools/ui/src/lib/utils/branching.ts +++ b/tools/ui/src/lib/utils/branching.ts @@ -105,18 +105,34 @@ export function filterByLeafNodeId( */ function findLeafNodeInMap( nodeMap: ReadonlyMap<string, DatabaseMessage>, - messageId: string + messageId: string, + leafCache?: Map<string, string> ): string { + const path: string[] = []; + let currentNode: DatabaseMessage | undefined = nodeMap.get(messageId); while (currentNode && currentNode.children.length > 0) { // Follow the last child (most recent branch) + const cached = leafCache?.get(currentNode.id); + + if (cached !== undefined) { + for (const id of path) leafCache?.set(id, cached); + + return cached; + } + + path.push(currentNode.id); const lastChildId = currentNode.children[currentNode.children.length - 1]; currentNode = nodeMap.get(lastChildId); } - return currentNode?.id ?? messageId; + const leafId = currentNode?.id ?? messageId; + + for (const id of path) leafCache?.set(id, leafId); + + return leafId; } /** @@ -176,7 +192,8 @@ export function findDescendantMessages( */ export function getMessageSiblings( nodeMap: ReadonlyMap<string, DatabaseMessage>, - messageId: string + messageId: string, + leafCache?: Map<string, string> ): ChatMessageSiblingInfo | null { const message = nodeMap.get(messageId); @@ -212,7 +229,7 @@ export function getMessageSiblings( // Convert sibling message IDs to their corresponding leaf node IDs // This allows navigation between different conversation branches const siblingLeafIds = siblingIds.map((siblingId: string) => - findLeafNodeInMap(nodeMap, siblingId) + findLeafNodeInMap(nodeMap, siblingId, leafCache) ); // Find current message's position among siblings const currentIndex = siblingIds.indexOf(messageId); @@ -236,9 +253,12 @@ export function buildSiblingInfoMap( ): Map<string, ChatMessageSiblingInfo> { const nodeMap = new Map(messages.map((msg) => [msg.id, msg] as const)); const siblingMap = new Map<string, ChatMessageSiblingInfo>(); + // Leaf walks repeat along the same child chains for every message; memoize + // them per build so each edge is walked once instead of O(messages^2) + const leafCache = new Map<string, string>(); for (const msg of messages) { - const info = getMessageSiblings(nodeMap, msg.id); + const info = getMessageSiblings(nodeMap, msg.id, leafCache); if (info) { siblingMap.set(msg.id, info); diff --git a/tools/ui/src/lib/utils/file-type.ts b/tools/ui/src/lib/utils/file-type.ts index fd8828fc1398..60f7c67d29f1 100644 --- a/tools/ui/src/lib/utils/file-type.ts +++ b/tools/ui/src/lib/utils/file-type.ts @@ -51,6 +51,7 @@ export function getFileTypeCategory(mimeType: string): FileTypeCategory | null { // Video case MimeTypeVideo.MP4: case MimeTypeVideo.OGG: + case MimeTypeVideo.WEBM: return FileTypeCategory.VIDEO; // PDF diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.ts index 079cdc871c60..721618c48fd2 100644 --- a/tools/ui/src/lib/utils/index.ts +++ b/tools/ui/src/lib/utils/index.ts @@ -285,7 +285,8 @@ export { extractSearchResults, extractSearchQuery, faviconForUrl, - isWebSearchToolName + isWebSearchToolName, + looksLikeSearchResult } from './search-results'; // Cache utilities diff --git a/tools/ui/src/lib/utils/parse-exec-shell-error.ts b/tools/ui/src/lib/utils/parse-exec-shell-error.ts index 42d2ee25413a..a7b2eb5c8ad6 100644 --- a/tools/ui/src/lib/utils/parse-exec-shell-error.ts +++ b/tools/ui/src/lib/utils/parse-exec-shell-error.ts @@ -3,8 +3,14 @@ export function parseExecShellCommandError( ): string | undefined { if (!toolResultString) return undefined; + // Exec results are usually large plain-text stdout; only a JSON object + // root can carry an error field, so skip the parse otherwise + const trimmed = toolResultString.trimStart(); + + if (trimmed[0] !== '{') return undefined; + try { - const parsed: unknown = JSON.parse(toolResultString); + const parsed: unknown = JSON.parse(trimmed); if ( parsed && diff --git a/tools/ui/src/lib/utils/parse-exec-shell-status.ts b/tools/ui/src/lib/utils/parse-exec-shell-status.ts index 1f7ec557edd1..71dd110bd149 100644 --- a/tools/ui/src/lib/utils/parse-exec-shell-status.ts +++ b/tools/ui/src/lib/utils/parse-exec-shell-status.ts @@ -15,15 +15,18 @@ export interface ExecShellExitStatus { } // Anchor to the absolute end so intermediate "[exit code: N]" string content -// (e.g. a shell echo) doesn't false-positive. +// (e.g. a shell echo) doesn't false-positive. The marker is at most ~50 chars +// with the timed-out suffix, so matching a tail slice keeps the cost constant +// for megabyte exec outputs instead of scanning the whole blob. const EXIT_CODE_TAIL_REGEX = /\[exit code: (-?\d+)\](?: \[exit due to timed out\])?\s*$/; +const EXIT_CODE_TAIL_SCAN = 128; export function parseExecShellCommandExitStatus( toolResultString: string | undefined ): ExecShellExitStatus | undefined { if (!toolResultString) return undefined; - const match = toolResultString.match(EXIT_CODE_TAIL_REGEX); + const match = toolResultString.slice(-EXIT_CODE_TAIL_SCAN).match(EXIT_CODE_TAIL_REGEX); if (!match) return undefined; diff --git a/tools/ui/src/lib/utils/search-results.ts b/tools/ui/src/lib/utils/search-results.ts index facf7766dfa2..0fe861d9465b 100644 --- a/tools/ui/src/lib/utils/search-results.ts +++ b/tools/ui/src/lib/utils/search-results.ts @@ -156,6 +156,20 @@ function parseChunk(chunk: string): SearchResult | null { return result; } +const EMPTY_SEARCH_RESULTS: SearchResult[] = []; + +/** + * Cheap prefilter for the wire format: a parseable result needs both a + * `Title:` and a `URL:` field line, so a blob missing either substring can + * never yield a result. Two substring scans cost far less than the + * line-split parse for the megabyte tool results exec and file tools emit. + */ +export function looksLikeSearchResult(text: string | undefined | null): boolean { + if (!text) return false; + + return text.includes('Title:') && text.includes('URL:'); +} + /** Bounded cache for extractSearchResults results. */ const SEARCH_RESULTS_CACHE_MAX_SIZE = 32; const searchResultsCache = new Map<string, SearchResult[]>(); @@ -168,7 +182,7 @@ const searchResultsCache = new Map<string, SearchResult[]>(); * tool result strings. */ export function extractSearchResults(text: string | undefined | null): SearchResult[] { - if (!text) return []; + if (!text || !looksLikeSearchResult(text)) return EMPTY_SEARCH_RESULTS; const cached = searchResultsCache.get(text); diff --git a/tools/ui/src/lib/utils/tool-call-meta.ts b/tools/ui/src/lib/utils/tool-call-meta.ts index b64bca7868e7..2c035446d3b3 100644 --- a/tools/ui/src/lib/utils/tool-call-meta.ts +++ b/tools/ui/src/lib/utils/tool-call-meta.ts @@ -4,6 +4,8 @@ // Each tool needs to surface fields like `error`, `result`, `bytes`, // `edits_applied` without repeating the try/JSON.parse/object guard inline. +import { JSON_OBJECT_OPEN } from '$lib/constants'; + /** * Parse a tool-result blob into a JSON object, or `null` if it isn't * one. Returns null for: @@ -16,8 +18,14 @@ export function tryParseToolResultObject( ): Record<string, unknown> | null { if (!toolResultString) return null; + // Tool results are usually large plain text (file contents, stdout); only + // a JSON object root can carry fields, so skip the parse otherwise + const trimmed = toolResultString.trimStart(); + + if (trimmed[0] !== JSON_OBJECT_OPEN) return null; + try { - const parsed: unknown = JSON.parse(toolResultString); + const parsed: unknown = JSON.parse(trimmed); if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { return parsed as Record<string, unknown>; diff --git a/tools/ui/src/routes/(chat)/+page.svelte b/tools/ui/src/routes/(chat)/+page.svelte index 08a6b11ad56d..53975d7b3d7f 100644 --- a/tools/ui/src/routes/(chat)/+page.svelte +++ b/tools/ui/src/routes/(chat)/+page.svelte @@ -3,7 +3,7 @@ import { page } from '$app/state'; import { DialogModelNotAvailable } from '$lib/components/app'; import { APP_NAME, URL_PARAMS } from '$lib/constants'; - import { chatStore, conversationsStore, modelsStore, serverStore } from '$lib/stores'; + import { conversationsStore, modelsStore, serverStore } from '$lib/stores'; import { onMount } from 'svelte'; let qParam = $derived(page.url.searchParams.get(URL_PARAMS.QUERY)); @@ -77,7 +77,6 @@ } conversationsStore.clearActiveConversation(); - chatStore.clearUIState(); await modelsStore.fetch(); diff --git a/tools/ui/src/routes/+layout.svelte b/tools/ui/src/routes/+layout.svelte index 38b65627155e..5de4f82c1cbe 100644 --- a/tools/ui/src/routes/+layout.svelte +++ b/tools/ui/src/routes/+layout.svelte @@ -325,14 +325,17 @@ }} /> - <div class="flex-1"> + <!-- min-w-0 lets the chat column shrink below its content width, so wide + code blocks and tables scroll inside their own containers instead of + stretching the page into a horizontal scrollbar --> + <div class="min-w-0 flex-1"> {@render children?.()} </div> </div> <ModeWatcher /> - <Toaster richColors /> + <Toaster closeButton richColors /> </Tooltip.Provider> <!-- PWA update prompt + version --> diff --git a/tools/ui/tests/stories/a11y/ChatScreenForm.a11y.stories.svelte b/tools/ui/tests/stories/a11y/ChatScreenForm.a11y.stories.svelte index c56bfd7678bd..826e0d4aa1eb 100644 --- a/tools/ui/tests/stories/a11y/ChatScreenForm.a11y.stories.svelte +++ b/tools/ui/tests/stories/a11y/ChatScreenForm.a11y.stories.svelte @@ -2,8 +2,25 @@ import { defineMeta } from '@storybook/addon-svelte-csf'; import ChatScreenForm from '$lib/components/app/chat/ChatScreen/ChatScreenForm.svelte'; import { ATTACHMENT_TOOLTIP_TEXT } from '$lib/constants'; + import { ServerRole } from '$lib/enums'; + import { serverStore } from '$lib/stores'; + import type { ApiLlamaCppServerProps } from '$lib/types'; import { expect, screen, waitFor } from 'storybook/test'; + /** + * The add menu mounts the reasoning submenu only outside router mode, and the + * dev server proxies /props to whichever server happens to be running, so pin + * the mode this story asserts instead of inheriting it from the environment. + */ + function pinSingleModelMode(): void { + serverStore.props = { + ...(serverStore.props ?? {}), + role: ServerRole.MODEL + } as ApiLlamaCppServerProps; + + serverStore.role = ServerRole.MODEL; + } + const { Story } = defineMeta({ component: ChatScreenForm, parameters: { @@ -38,6 +55,8 @@ args={{ class: 'max-w-[56rem] w-[calc(100vw-2rem)]' }} name="AddDropdownFocusesFirstEnabled" play={async ({ canvas, userEvent }) => { + pinSingleModelMode(); + const trigger = await canvas.findByRole('button', { name: ATTACHMENT_TOOLTIP_TEXT }); trigger.focus(); @@ -45,7 +64,7 @@ await screen.findByRole('menu'); await waitFor(() => { - expect(document.activeElement).toHaveTextContent('Add files'); + expect(document.activeElement).toHaveTextContent('Reasoning'); }); }} /> diff --git a/tools/ui/tests/unit/agentic-sections.test.ts b/tools/ui/tests/unit/agentic-sections.test.ts index 4096a1710709..fdb3b221780c 100644 --- a/tools/ui/tests/unit/agentic-sections.test.ts +++ b/tools/ui/tests/unit/agentic-sections.test.ts @@ -290,3 +290,114 @@ describe('hasAgenticContent', () => { expect(hasAgenticContent(msg)).toBe(false); }); }); + +// The turn-section cache: completed turns are immutable, so repeated +// derivations return the same section objects - which is what keeps tool +// block props stable while another turn streams. Every field the cache +// compares must invalidate it; a miss here renders stale content. + +describe('completed turn section reuse', () => { + const toolCallsJson = JSON.stringify([ + { function: { arguments: '{"path":"/a"}', name: 'test' }, id: 'call_1', type: 'function' } + ]); + + function makeSession() { + return { + anchor: makeAssistant({ + content: 'answer', + reasoningContent: 'thinking', + toolCalls: toolCallsJson + }), + tools: [makeToolMsg({ content: 'tool result', extra: [{ type: 'file' } as never] })] + }; + } + + it('returns the same section objects for unchanged inputs', () => { + const { anchor, tools } = makeSession(); + const first = deriveAgenticSections(anchor, tools, [], false); + const second = deriveAgenticSections(anchor, tools, [], false); + + expect(second[0]).toBe(first[0]); + expect(second[1]).toBe(first[1]); + }); + + it('recomputes when the assistant content changes', () => { + const { anchor, tools } = makeSession(); + const first = deriveAgenticSections(anchor, tools, [], false); + + anchor.content = 'edited'; + const second = deriveAgenticSections(anchor, tools, [], false); + + expect(second).not.toBe(first); + expect(second.some((s) => s.type === AgenticSectionType.TEXT && s.content === 'edited')).toBe( + true + ); + }); + + it('recomputes when reasoning content changes', () => { + const { anchor, tools } = makeSession(); + const first = deriveAgenticSections(anchor, tools, [], false); + + anchor.reasoningContent = 'new thinking'; + const second = deriveAgenticSections(anchor, tools, [], false); + + expect(second).not.toBe(first); + }); + + it('recomputes when toolCalls change', () => { + const { anchor, tools } = makeSession(); + const first = deriveAgenticSections(anchor, tools, [], false); + + anchor.toolCalls = '[]'; + const second = deriveAgenticSections(anchor, tools, [], false); + + expect(second).not.toBe(first); + }); + + it('recomputes when a tool result or its extras change', () => { + const { anchor, tools } = makeSession(); + const first = deriveAgenticSections(anchor, tools, [], false); + + tools[0].content = 'new tool result'; + expect(deriveAgenticSections(anchor, tools, [], false)).not.toBe(first); + + const firstAfterContent = deriveAgenticSections(anchor, tools, [], false); + + tools[0].extra = [{ type: 'image' } as never]; + expect(deriveAgenticSections(anchor, tools, [], false)).not.toBe(firstAfterContent); + }); + + it('never reuses the streaming turn', () => { + const { anchor, tools } = makeSession(); + const first = deriveAgenticSections(anchor, tools, [], true); + const second = deriveAgenticSections(anchor, tools, [], true); + + expect(second).not.toBe(first); + }); + + it('keeps completed turns stable while the last turn streams', () => { + const anchor = makeAssistant({ + content: 'turn one', + id: 'ast-1', + toolCalls: JSON.stringify([ + { function: { arguments: '{}', name: 'test' }, id: 'call_1', type: 'function' } + ]) + }); + const continuation = makeAssistant({ content: 'turn two', id: 'ast-2' }); + const tools = [ + makeToolMsg({ content: 'r1', id: 'tool-1', toolCallId: 'call_1' }), + continuation, + makeToolMsg({ content: 'r2', id: 'tool-2', toolCallId: 'call_2' }) + ]; + const first = deriveAgenticSections(anchor, tools, [], true); + const second = deriveAgenticSections(anchor, tools, [], true); + + // turn one is complete: identical section objects across derivations + expect(second.slice(0, 2)).toEqual(first.slice(0, 2)); + expect(second[0]).toBe(first[0]); + expect(second[1]).toBe(first[1]); + + // the streaming last turn recomputed: fresh section objects + expect(second[second.length - 1]).not.toBe(first[first.length - 1]); + }); +}); diff --git a/tools/ui/tests/unit/branching.test.ts b/tools/ui/tests/unit/branching.test.ts new file mode 100644 index 000000000000..8a752ae2f1ba --- /dev/null +++ b/tools/ui/tests/unit/branching.test.ts @@ -0,0 +1,95 @@ +// Sibling-info correctness for buildSiblingInfoMap, including the memoized +// leaf resolution. A wrong leaf id here breaks branch navigation, so the +// deep-chain and multi-branch cases below pin the resolution down. + +import { MessageRole, MessageType } from '$lib/enums'; +import type { DatabaseMessage } from '$lib/types/database'; +import { buildSiblingInfoMap, findLeafNode } from '$lib/utils/branching'; +import { describe, expect, it } from 'vitest'; + +function msg(id: string, parent: string | null, children: string[] = []): DatabaseMessage { + return { + children, + content: '', + convId: 'c1', + id, + parent, + role: MessageRole.USER, + timestamp: 0, + type: MessageType.TEXT + } as DatabaseMessage; +} + +/** root -> m1 -> ... -> m depth, each node with a single child. */ +function linearChain(depth: number): DatabaseMessage[] { + const messages = [msg('m0', null, ['m1'])]; + + for (let i = 1; i <= depth; i++) { + messages.push(msg(`m${i}`, `m${i - 1}`, i < depth ? [`m${i + 1}`] : [])); + } + + return messages; +} + +describe('buildSiblingInfoMap', () => { + it('resolves the deepest leaf for every node of a long single chain', () => { + const messages = linearChain(50); + const map = buildSiblingInfoMap(messages); + const leafId = messages[messages.length - 1].id; + + // every non-root message of the chain is an only child, and its + // navigation target is the chain's deepest leaf + for (const m of messages.slice(1)) { + const info = map.get(m.id); + + expect(info?.totalSiblings).toBe(1); + expect(info?.siblingIds).toEqual([leafId]); + } + }); + + it('reports sibling position and leaf targets on a branched tree', () => { + // m0 -> m1, m4 ; m1 -> m2 ; m2 -> m3, m6 ; m4 -> m5 + const root = msg('m0', null, ['m1', 'm4']); + const m1 = msg('m1', 'm0', ['m2']); + const m2 = msg('m2', 'm1', ['m3', 'm6']); + const m3 = msg('m3', 'm2'); + const m4 = msg('m4', 'm0', ['m5']); + const m5 = msg('m5', 'm4'); + const m6 = msg('m6', 'm2'); + const map = buildSiblingInfoMap([root, m1, m2, m3, m4, m5, m6]); + + // m1 and m4 share the root as parent; their nav targets are the + // leaves of their subtrees ( m6 for the first branch, m5 for the second ) + expect(map.get(m1.id)).toMatchObject({ + currentIndex: 0, + siblingIds: [m6.id, m5.id], + totalSiblings: 2 + }); + expect(map.get(m4.id)).toMatchObject({ + currentIndex: 1, + siblingIds: [m6.id, m5.id], + totalSiblings: 2 + }); + + // m3 and m6 are siblings under m2; both are leaves + expect(map.get(m3.id)?.siblingIds).toEqual([m3.id, m6.id]); + expect(map.get(m6.id)?.currentIndex).toBe(1); + + // the root has no parent and reports itself + expect(map.get(root.id)).toMatchObject({ + currentIndex: 0, + siblingIds: [root.id], + totalSiblings: 1 + }); + }); + + it('agrees with findLeafNode for arbitrary nodes', () => { + const messages = linearChain(20); + const leafId = messages[messages.length - 1].id; + + // every node of the chain resolves to the deepest leaf + for (const m of messages) { + expect(findLeafNode(messages, m.id), `leaf of ${m.id}`).toBe(leafId); + } + }); +}); diff --git a/tools/ui/tests/unit/conversations-store.test.ts b/tools/ui/tests/unit/conversations-store.test.ts new file mode 100644 index 000000000000..e06546597bbe --- /dev/null +++ b/tools/ui/tests/unit/conversations-store.test.ts @@ -0,0 +1,90 @@ +// Field updates to the active conversation must keep the object identity +// stable: effects that track the identity ( the chat screen's sibling-info +// refresh ) refire on every identity change, which used to trigger a full +// message refetch on every send and tool result. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('$lib/services/database.service', () => ({ + DatabaseService: { + getConversation: vi.fn(), + getConversationMessages: vi.fn(), + updateConversation: vi.fn(), + updateCurrentNode: vi.fn() + } +})); + +import { DatabaseService } from '$lib/services/database.service'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; +import type { DatabaseConversation, DatabaseMessage } from '$lib/types/database'; + +const getConversationMock = vi.mocked(DatabaseService.getConversation); +const getMessagesMock = vi.mocked(DatabaseService.getConversationMessages); +const updateCurrentNodeMock = vi.mocked(DatabaseService.updateCurrentNode); + +function makeConversation(overrides: Partial<DatabaseConversation> = {}): DatabaseConversation { + return { + currNode: 'node-1', + id: 'conv-1', + lastModified: 1000, + name: 'conversation', + ...overrides + }; +} + +async function loadActive(conversation: DatabaseConversation, messages: DatabaseMessage[]) { + getConversationMock.mockResolvedValue(conversation); + getMessagesMock.mockResolvedValue(messages); + + expect(await conversationsStore.loadConversation(conversation.id)).toBe(true); +} + +beforeEach(() => { + getConversationMock.mockReset(); + getMessagesMock.mockReset(); + updateCurrentNodeMock.mockReset(); + updateCurrentNodeMock.mockResolvedValue(undefined); + vi.mocked(DatabaseService.updateConversation).mockReset(); + vi.mocked(DatabaseService.updateConversation).mockResolvedValue(undefined); +}); + +describe('active conversation identity', () => { + it('hands the load read off exactly once', async () => { + await loadActive(makeConversation(), []); + + expect(conversationsStore.consumeLastLoadedMessages('conv-1')).toEqual([]); + // a second consume is a miss: branch actions must fall back to a refetch + expect(conversationsStore.consumeLastLoadedMessages('conv-1')).toBeNull(); + }); + + it('writes currNode in place on updateCurrentNode', async () => { + await loadActive(makeConversation(), []); + const before = conversationsStore.activeConversation; + + await conversationsStore.updateCurrentNode('node-2'); + + expect(conversationsStore.activeConversation).toBe(before); + expect(conversationsStore.activeConversation?.currNode).toBe('node-2'); + }); + + it('writes renamed and pinned fields in place on applyConversationUpdate', async () => { + await loadActive(makeConversation(), []); + const before = conversationsStore.activeConversation; + + conversationsStore.applyConversationUpdate('conv-1', { name: 'renamed', pinned: true }); + + expect(conversationsStore.activeConversation).toBe(before); + expect(conversationsStore.activeConversation?.name).toBe('renamed'); + expect(conversationsStore.activeConversation?.pinned).toBe(true); + }); + + it('writes lastModified in place on updateConversationTimestamp', async () => { + await loadActive(makeConversation(), []); + const before = conversationsStore.activeConversation; + + conversationsStore.updateConversationTimestamp('conv-1'); + + expect(conversationsStore.activeConversation).toBe(before); + expect(conversationsStore.activeConversation?.lastModified).toBeGreaterThan(1000); + }); +}); diff --git a/tools/ui/tests/unit/parse-exec-shell-status.test.ts b/tools/ui/tests/unit/parse-exec-shell-status.test.ts index ed499d078e8e..7e22bf9eeb8d 100644 --- a/tools/ui/tests/unit/parse-exec-shell-status.test.ts +++ b/tools/ui/tests/unit/parse-exec-shell-status.test.ts @@ -71,3 +71,21 @@ describe('isExitCodeSummaryLine', () => { expect(isExitCodeSummaryLine('[exit code: 7]', undefined)).toBe(false); }); }); + +describe('parseExecShellCommandExitStatus tail scan', () => { + it('finds the marker at the end of a blob larger than the tail window', () => { + // the parser matches only the last ~128 chars; a marker past that + // window must still parse, and an earlier fake must not match + const blob = `${'the shell prints [exit code: 1] mid-stream\n'.repeat(2000)}[exit code: 0]`; + const status = parseExecShellCommandExitStatus(blob); + + expect(status?.code).toBe(0); + expect(status?.timedOut).toBe(false); + }); + + it('keeps rejecting markers that are not at the absolute end', () => { + const blob = `${'stdout\n'.repeat(2000)}[exit code: 0]\nsome trailing log line`; + + expect(parseExecShellCommandExitStatus(blob)).toBeUndefined(); + }); +}); diff --git a/tools/ui/tests/unit/search-results.test.ts b/tools/ui/tests/unit/search-results.test.ts index c168dec25829..561ab935a819 100644 --- a/tools/ui/tests/unit/search-results.test.ts +++ b/tools/ui/tests/unit/search-results.test.ts @@ -2,7 +2,8 @@ import { extractSearchQuery, extractSearchResults, faviconForUrl, - isWebSearchToolName + isWebSearchToolName, + looksLikeSearchResult } from '$lib/utils/search-results'; import { describe, expect, it } from 'vitest'; @@ -119,3 +120,27 @@ describe('isWebSearchToolName', () => { expect(isWebSearchToolName('exec_shell_command')).toBe(false); }); }); + +describe('extractSearchResults prefilter', () => { + it('returns the shared empty array for blobs without the wire format', () => { + // exec/file tool results never carry Title:/URL: field lines; the + // cheap prefilter must skip the line-split parse for them + const stdout = `${'make[1]: entering directory\n'.repeat(5000)}`; + + expect(extractSearchResults(stdout)).toEqual([]); + }); + + it('returns an empty result when only one required field is present', () => { + expect(extractSearchResults('URL: https://example.com')).toEqual([]); + expect(extractSearchResults('Title: only a title')).toEqual([]); + }); +}); + +describe('looksLikeSearchResult', () => { + it('requires both Title and URL field markers', () => { + expect(looksLikeSearchResult('Title: a\nURL: https://b')).toBe(true); + expect(looksLikeSearchResult('URL: https://b')).toBe(false); + expect(looksLikeSearchResult('plain stdout')).toBe(false); + expect(looksLikeSearchResult(undefined)).toBe(false); + }); +}); diff --git a/tools/ui/tests/unit/tool-call-meta.test.ts b/tools/ui/tests/unit/tool-call-meta.test.ts index bb28e3830cdf..f94d2279feb3 100644 --- a/tools/ui/tests/unit/tool-call-meta.test.ts +++ b/tools/ui/tests/unit/tool-call-meta.test.ts @@ -28,3 +28,15 @@ describe('tryParseToolResultObject', () => { expect(tryParseToolResultObject('{bad')).toBeNull(); }); }); + +describe('tryParseToolResultObject gating', () => { + it('parses JSON objects that start after leading whitespace', () => { + expect(tryParseToolResultObject('\n {"result":"ok"}')).toEqual({ result: 'ok' }); + }); + + it('skips the parse for large plain-text results', () => { + // most tool results are file contents or stdout; the gate avoids a + // doomed JSON.parse over the whole blob + expect(tryParseToolResultObject(`${'stdout line\n'.repeat(2000)}`)).toBeNull(); + }); +}); diff --git a/tools/ui/tests/unit/tool-calls.test.ts b/tools/ui/tests/unit/tool-calls.test.ts index f84a2405ecb1..a2274f9d2803 100644 --- a/tools/ui/tests/unit/tool-calls.test.ts +++ b/tools/ui/tests/unit/tool-calls.test.ts @@ -1,5 +1,8 @@ import { parseToolArgs } from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/_shared'; -import { parseEditFileMeta } from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/edit-file'; +import { + parseEditFileMeta, + parseEditFileTitleMeta +} from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/edit-file'; import { parseExecShellCommandMeta } from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/exec-shell-command'; import { parseFileGlobSearchMeta } from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/file-glob-search'; import { parseGrepSearchMeta } from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/grep-search'; @@ -7,10 +10,10 @@ import { parseReadFileMeta } from '$lib/components/app/chat/ChatMessages/ChatMes import { parseRunJavascriptMeta } from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/run-javascript'; import { parseWriteFileMeta, - type WriteFileMeta + parseWriteFileTitleMeta } from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/write-file'; import { AgenticSectionType, BuiltInTool } from '$lib/enums'; -import type { AgenticSection } from '$lib/types'; +import type { AgenticSection, WriteFileMeta } from '$lib/types'; import { abbreviateHome, formatCwdMessage, lastPathSegment, parseCwdMessage } from '$lib/utils'; import { describe, expect, it } from 'vitest'; @@ -223,6 +226,113 @@ describe('parseWriteFileMeta', () => { }); }); +describe('parseWriteFileTitleMeta', () => { + it('matches the full meta for path, language and result fields', () => { + const args = JSON.stringify({ content: 'x'.repeat(50_000), path: '/foo.ts' }); + const toolResult = '{"result":"wrote","bytes":42}'; + const section = makeSection( + { toolArgs: args, toolName: BuiltInTool.SERVER_WRITE_FILE, toolResult }, + BuiltInTool.SERVER_WRITE_FILE + ); + const full = parseWriteFileMeta(section); + const title = parseWriteFileTitleMeta(section); + + expect(title?.filePath).toBe(full?.filePath); + expect(title?.fileName).toBe(full?.fileName); + expect(title?.language).toBe(full?.language); + expect(title?.bytesWritten).toBe(full?.bytesWritten); + expect(title?.resultMessage).toBe(full?.resultMessage); + expect(title?.errorMessage).toBe(full?.errorMessage); + }); + + it('extracts a path with escaped characters without parsing the content blob', () => { + const section = makeSection( + { + toolArgs: '{"path":"/a\\nb\\"c/d.ts","content":"x"}', + toolName: BuiltInTool.SERVER_WRITE_FILE + }, + BuiltInTool.SERVER_WRITE_FILE + ); + + expect(parseWriteFileTitleMeta(section)?.filePath).toBe('/a\nb"c/d.ts'); + }); + + it('falls back to the full parse for args the extractor can not see', () => { + const section = makeSection( + { + // key written with an escaped unicode escape sequence in the name + toolArgs: '{"\\u0070ath":"/foo.ts","content":"x"}', + toolName: BuiltInTool.SERVER_WRITE_FILE + }, + BuiltInTool.SERVER_WRITE_FILE + ); + + expect(parseWriteFileTitleMeta(section)?.filePath).toBe('/foo.ts'); + }); + + it('accepts partial args like the full parser', () => { + const section = makeSection( + { toolArgs: '{"path":"/foo.t', toolName: BuiltInTool.SERVER_WRITE_FILE }, + BuiltInTool.SERVER_WRITE_FILE + ); + + expect(parseWriteFileTitleMeta(section)?.filePath).toBe('/foo.t'); + }); + + it('returns null for sections with a different tool name', () => { + expect( + parseWriteFileTitleMeta( + makeSection({ + toolArgs: '{"path":"/x","content":"y"}', + toolName: BuiltInTool.SERVER_READ_FILE + }) + ) + ).toBeNull(); + }); +}); + +describe('parseEditFileTitleMeta', () => { + it('matches the full meta for path and result fields', () => { + const section = makeSection( + { + toolArgs: '{"path":"/foo.ts","edits":[{"old_text":"a","new_text":"b"}]}' + ' '.repeat(0), + toolName: BuiltInTool.SERVER_EDIT_FILE, + toolResult: '{"result":"ok","edits_applied":1}' + }, + BuiltInTool.SERVER_EDIT_FILE + ); + const full = parseEditFileMeta(section); + const title = parseEditFileTitleMeta(section); + + expect(title?.filePath).toBe(full?.filePath); + expect(title?.fileName).toBe(full?.fileName); + expect(title?.editsApplied).toBe(full?.editsApplied); + expect(title?.resultMessage).toBe(full?.resultMessage); + expect(title?.errorMessage).toBe(full?.errorMessage); + }); + + it('surfaces errorMessage from the result blob without parsing args', () => { + const section = makeSection( + { + toolArgs: '{"path":"/foo.ts","edits":[]}', + toolName: BuiltInTool.SERVER_EDIT_FILE, + toolResult: '{"error":"permission denied"}' + }, + BuiltInTool.SERVER_EDIT_FILE + ); + + expect(parseEditFileTitleMeta(section)?.errorMessage).toBe('permission denied'); + }); + + it('returns null when args have no path-like field', () => { + expect( + parseEditFileTitleMeta( + makeSection({ toolArgs: '{"edits":[]}', toolName: BuiltInTool.SERVER_EDIT_FILE }) + ) + ).toBeNull(); + }); +}); + describe('parseEditFileMeta', () => { it('parses edits array and applies editsApplied from the result', () => { const section = makeSection( diff --git a/tools/ui/ui.cpp.in b/tools/ui/ui.cpp.in new file mode 100644 index 000000000000..7f91ef2a224e --- /dev/null +++ b/tools/ui/ui.cpp.in @@ -0,0 +1,36 @@ +// Generated by scripts/ui-assets.cmake - do not edit. + +#include "ui.h" + +@ASSET_ARRAYS@ +#if defined(LLAMA_UI_HAS_ASSETS) +static const std::array<llama_ui_asset, @N_ASSETS@> g_assets = {{ +@ASSET_TABLE@ +}}; +#endif + +const llama_ui_asset * llama_ui_find_asset(const std::string & name) { +#if defined(LLAMA_UI_HAS_ASSETS) + for (const auto & a : g_assets) { + if (a.name == name) { + return &a; + } + } +#else + (void) name; +#endif + return nullptr; +} + +const std::array<llama_ui_asset, @N_ASSETS@> & llama_ui_get_assets() { +#if defined(LLAMA_UI_HAS_ASSETS) + return g_assets; +#else + static const std::array<llama_ui_asset, 0> empty{}; + return empty; +#endif +} + +bool llama_ui_use_gzip() { + return @USE_GZIP@; +} diff --git a/tools/ui/ui.h.in b/tools/ui/ui.h.in new file mode 100644 index 000000000000..4555b0dd5295 --- /dev/null +++ b/tools/ui/ui.h.in @@ -0,0 +1,21 @@ +// Generated by scripts/ui-assets.cmake - do not edit. + +#pragma once + +#include <array> +#include <string> + +// Defined as 1 only when assets were embedded (tools/server checks defined()). +#cmakedefine LLAMA_UI_HAS_ASSETS 1 + +struct llama_ui_asset { + std::string name; + const unsigned char * data; + std::size_t size; + std::string etag; + std::string type; +}; + +const llama_ui_asset * llama_ui_find_asset(const std::string & name); +bool llama_ui_use_gzip(); +const std::array<llama_ui_asset, @N_ASSETS@> & llama_ui_get_assets(); diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp index 7fd10b3939bc..df79b3c7d91a 100644 --- a/vendor/cpp-httplib/httplib.cpp +++ b/vendor/cpp-httplib/httplib.cpp @@ -912,17 +912,42 @@ bool write_websocket_frame(Stream &strm, ws::Opcode opcode, namespace ws { namespace impl { -bool read_websocket_frame(Stream &strm, Opcode &opcode, - std::string &payload, bool &fin, - bool expect_masked, size_t max_len) { - // Read first 2 bytes +// Read exactly `size` bytes. Stream::read may return less than asked for -- it +// hands back whatever its buffer already holds -- so every multi-byte field has +// to loop. Reading a 2-byte header with a single read() fails whenever the +// header straddles the read buffer's boundary. +// +// Timeout is reported only when nothing at all was consumed. Once a byte has +// been taken the stream sits mid-field and cannot be resumed, so a timeout +// there is a failure like any other. (When read() fails it always records why, +// so the error belongs to this call and not to an earlier one.) +FrameRead read_exact(Stream &strm, void *buf, size_t size) { + auto p = static_cast<char *>(buf); + size_t total = 0; + while (total < size) { + auto n = strm.read(p + total, size - total); + if (n <= 0) { + auto timed_out = total == 0 && strm.get_error() == Error::Timeout; + return timed_out ? FrameRead::Timeout : FrameRead::Fail; + } + total += static_cast<size_t>(n); + } + return FrameRead::Ok; +} + +FrameRead read_websocket_frame(Stream &strm, Opcode &opcode, + std::string &payload, bool &fin, + bool expect_masked, size_t max_len) { + // Read first 2 bytes. This is the only read that may report a timeout: it + // sits on a frame boundary, where nothing has been consumed yet. uint8_t header[2]; - if (strm.read(reinterpret_cast<char *>(header), 2) != 2) { return false; } + FrameRead first = read_exact(strm, header, 2); + if (first != FrameRead::Ok) { return first; } fin = (header[0] & 0x80) != 0; // RSV1, RSV2, RSV3 must be 0 when no extension is negotiated - if (header[0] & 0x70) { return false; } + if (header[0] & 0x70) { return FrameRead::Fail; } opcode = static_cast<Opcode>(header[0] & 0x0F); bool masked = (header[1] & 0x80) != 0; @@ -932,46 +957,44 @@ bool read_websocket_frame(Stream &strm, Opcode &opcode, // MUST have a payload length of 125 bytes or less bool is_control = (static_cast<uint8_t>(opcode) & 0x08) != 0; if (is_control) { - if (!fin) { return false; } - if (payload_len > 125) { return false; } + if (!fin) { return FrameRead::Fail; } + if (payload_len > 125) { return FrameRead::Fail; } } - if (masked != expect_masked) { return false; } + if (masked != expect_masked) { return FrameRead::Fail; } // Extended payload length if (payload_len == 126) { uint8_t ext[2]; - if (strm.read(reinterpret_cast<char *>(ext), 2) != 2) { return false; } + if (read_exact(strm, ext, 2) != FrameRead::Ok) { return FrameRead::Fail; } payload_len = (static_cast<uint64_t>(ext[0]) << 8) | ext[1]; } else if (payload_len == 127) { uint8_t ext[8]; - if (strm.read(reinterpret_cast<char *>(ext), 8) != 8) { return false; } + if (read_exact(strm, ext, 8) != FrameRead::Ok) { return FrameRead::Fail; } // RFC 6455 Section 5.2: the most significant bit MUST be 0 - if (ext[0] & 0x80) { return false; } + if (ext[0] & 0x80) { return FrameRead::Fail; } payload_len = 0; for (int i = 0; i < 8; i++) { payload_len = (payload_len << 8) | ext[i]; } } - if (payload_len > max_len) { return false; } + if (payload_len > max_len) { return FrameRead::Fail; } // Read mask key if present uint8_t mask_key[4] = {0}; if (masked) { - if (strm.read(reinterpret_cast<char *>(mask_key), 4) != 4) { return false; } + if (read_exact(strm, mask_key, 4) != FrameRead::Ok) { + return FrameRead::Fail; + } } // Read payload payload.resize(static_cast<size_t>(payload_len)); - if (payload_len > 0) { - size_t total_read = 0; - while (total_read < payload_len) { - auto n = strm.read(&payload[total_read], - static_cast<size_t>(payload_len - total_read)); - if (n <= 0) { return false; } - total_read += static_cast<size_t>(n); - } + if (payload_len > 0 && + read_exact(strm, &payload[0], static_cast<size_t>(payload_len)) != + FrameRead::Ok) { + return FrameRead::Fail; } // Unmask if needed @@ -981,7 +1004,7 @@ bool read_websocket_frame(Stream &strm, Opcode &opcode, } } - return true; + return FrameRead::Ok; } } // namespace impl @@ -1225,6 +1248,13 @@ bool parse_trailers(stream_line_reader &line_reader, Headers &dest, // to look up. split(trailer_header.data(), trailer_header.data() + trailer_header.size(), ',', [&](const char *b, const char *e) { + // A legitimate message declares only a handful of trailers. Cap the + // set so a peer cannot grow it without bound: an oversized set only + // arises from an attempt to force many colliding names into + // quadratic lookups (case_ignore::hash is unkeyed). + if (declared_trailers.size() >= CPPHTTPLIB_HEADER_MAX_COUNT) { + return; + } std::string key(b, e); if (prohibited_trailers.find(key) == prohibited_trailers.end()) { declared_trailers.insert(key); @@ -1235,6 +1265,8 @@ bool parse_trailers(stream_line_reader &line_reader, Headers &dest, size_t trailer_header_count = 0; while (strcmp(line_reader.ptr(), "\r\n") != 0) { if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; } + // Count every received trailer field, not only the declared ones stored in + // dest, so undeclared fields cannot keep this loop running past the limit. if (trailer_header_count >= CPPHTTPLIB_HEADER_MAX_COUNT) { return false; } constexpr auto line_terminator_len = 2; @@ -1247,12 +1279,13 @@ bool parse_trailers(stream_line_reader &line_reader, Headers &dest, if (declared_trailers.find(key) != declared_trailers.end()) { dest.emplace(key, val); - trailer_header_count++; } })) { return false; } + trailer_header_count++; + if (!line_reader.getline()) { return false; } } @@ -1728,7 +1761,9 @@ ssize_t select_impl(socket_t sock, short events, time_t sec, pfd.events = events; pfd.revents = 0; - auto timeout = static_cast<int>(sec * 1000 + usec / 1000); + // A negative timeout waits forever, poll's own convention. 0 keeps meaning + // "return immediately", which callers here rely on to probe a socket. + auto timeout = sec < 0 ? -1 : static_cast<int>(sec * 1000 + usec / 1000); return handle_EINTR([&]() { return poll_wrapper(&pfd, 1, timeout); }); } @@ -1810,8 +1845,11 @@ class SocketStream final : public Stream { bool ensure_readable(); socket_t sock_; - time_t read_timeout_sec_; - time_t read_timeout_usec_; + // Atomic because ws::WebSocket::set_read_timeout() reaches this from another + // thread while a read is in flight -- that is the point of it, for a caller + // holding one connection and wanting control back to send on it. + std::atomic<time_t> read_timeout_sec_; + std::atomic<time_t> read_timeout_usec_; time_t write_timeout_sec_; time_t write_timeout_usec_; time_t max_timeout_msec_; @@ -2204,12 +2242,10 @@ int getaddrinfo_with_timeout(const char *node, const char *service, // actually finish before letting the stack frame go. The trade-off is that // a wedged DNS server can hold this thread for the system resolver timeout // (~30s by default) past the caller's connection timeout. - struct gaicb request {}; + struct gaicb request{}; struct gaicb *requests[1] = {&request}; - struct sigevent sevp {}; - struct timespec timeout { - timeout_sec, 0 - }; + struct sigevent sevp{}; + struct timespec timeout{timeout_sec, 0}; request.ar_name = node; request.ar_service = service; @@ -2948,8 +2984,21 @@ EncodingType encoding_type(const Request &req, return best; } +// `content_type` is taken separately because a file-backed response has not +// been given one yet when its coding has to be decided. +EncodingType encoding_type(const Request &req, const Response &res, + const std::string &content_type) { + // The response already names a content coding of its own: a handler serving + // a body it encoded itself (pre-compressed static assets, say), or a mount + // point whose headers name the coding its files are stored in. Applying one + // on top of that would double-encode the body and append a second + // `Content-Encoding` field line. + if (res.has_header("Content-Encoding")) { return EncodingType::None; } + return encoding_type(req, content_type); +} + EncodingType encoding_type(const Request &req, const Response &res) { - return encoding_type(req, res.get_header_value("Content-Type")); + return encoding_type(req, res, res.get_header_value("Content-Type")); } std::unique_ptr<compressor> make_compressor(EncodingType type) { @@ -3677,6 +3726,17 @@ bool is_chunked_transfer_encoding(const Headers &headers) { return case_ignore::equal(last_coding, "chunked"); } +bool has_conflicting_content_length(const Headers &headers) { + // RFC 9112 §6.3: a message carrying both Transfer-Encoding and a non-zero + // Content-Length is framed ambiguously. The body readers here delimit it by + // the transfer coding and drop Content-Length, while an intermediary may do + // the reverse, so the two disagree on where the body ends and a reused + // connection is desynchronised (request/response smuggling). Content-Length: + // 0 is tolerated for compatibility with existing peers. + return has_header(headers, "Transfer-Encoding") && + get_header_value_u64(headers, "Content-Length", 0, 0) > 0; +} + template <typename T, typename U> bool prepare_content_receiver(T &x, int &status, ContentReceiverWithProgress receiver, @@ -3794,11 +3854,13 @@ bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status, ssize_t write_request_line(Stream &strm, const std::string &method, const std::string &path) { - // A request target must not carry CR/LF (or other control octets); otherwise - // a value smuggled into it splits the request line and injects headers or a - // whole request. The same field-value check already guards header values in - // check_and_write_headers and the request target in - // perform_websocket_handshake; apply it here too. + // Neither the method nor the request target may carry CR/LF (or other + // control octets); otherwise a value smuggled into either splits the request + // line and injects headers or a whole request. The method must be a token + // (RFC 9110 Section 9.1), which also rejects an empty method and embedded + // spaces. The target gets the same field-value check that already guards + // header values in check_and_write_headers. + if (!fields::is_token(method)) { return -1; } if (!fields::is_field_value(path)) { return -1; } std::string s = method; @@ -4035,7 +4097,7 @@ void set_file_content_provider(Response &res, return true; }); - res.file_content_encoding_ = encoding; + res.content_coding_ = encoding; } template <typename T, typename U> @@ -4361,13 +4423,20 @@ bool parse_range_header(const std::string &s, Ranges &ranges) try { ssize_t first = -1; if (!lhs.empty()) { - ssize_t v; - auto res = detail::from_chars(lhs.data(), lhs.data() + lhs.size(), v); - if (res.ec == std::errc{}) { first = v; } + // Reject an overflowing first-byte-pos; treating it as absent (-1) + // would turn the range into a suffix range. + auto res = + detail::from_chars(lhs.data(), lhs.data() + lhs.size(), first); + if (res.ec != std::errc{}) { + all_valid_ranges = false; + return; + } } ssize_t last = -1; if (!rhs.empty()) { + // An overflowing last-byte-pos is past any content length, so keeping + // -1 ("remainder", RFC 9110 14.1.2) is correct here. ssize_t v; auto res = detail::from_chars(rhs.data(), rhs.data() + rhs.size(), v); if (res.ec == std::errc{}) { last = v; } @@ -6902,7 +6971,7 @@ void Response::set_content(const char *s, size_t n, auto rng = headers.equal_range("Content-Type"); headers.erase(rng.first, rng.second); set_header("Content-Type", content_type); - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content(const std::string &s, @@ -6917,7 +6986,7 @@ void Response::set_content(std::string &&s, auto rng = headers.equal_range("Content-Type"); headers.erase(rng.first, rng.second); set_header("Content-Type", content_type); - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content_provider( @@ -6928,7 +6997,7 @@ void Response::set_content_provider( if (in_length > 0) { content_provider_ = std::move(provider); } content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = false; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content_provider( @@ -6939,7 +7008,7 @@ void Response::set_content_provider( content_provider_ = detail::ContentProviderAdapter(std::move(provider)); content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = false; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_chunked_content_provider( @@ -6950,7 +7019,7 @@ void Response::set_chunked_content_provider( content_provider_ = detail::ContentProviderAdapter(std::move(provider)); content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = true; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_file_content(const std::string &path, @@ -7991,12 +8060,19 @@ ssize_t WebSocketSSLStream::read(char *ptr, size_t size) { needs_readable || (err.code == tls::ErrorCode::SyscallError && WSAGetLastError() == WSAETIMEDOUT); #endif - if (!needs_readable && err.code != tls::ErrorCode::WantWrite) { return -1; } + if (!needs_readable && err.code != tls::ErrorCode::WantWrite) { + error_ = Error::Read; + return -1; + } if (!(needs_readable ? wait_readable() : wait_writable())) { error_ = Error::Timeout; return -1; } } + // Out of retries. Recording a reason matters: a caller that reads get_error() + // to tell a timeout from a close would otherwise see whatever the previous + // failure left behind (error_ is never cleared on success). + error_ = Error::Read; return -1; } @@ -8562,7 +8638,10 @@ bool Server::write_response_core(Stream &strm, bool close_connection, // Prepare additional headers if (close_connection || detail::has_header_token(req.headers, "Connection", "close") || - 400 <= res.status) { // Don't leave connections open after errors + 400 <= res.status || // Don't leave connections open after errors + // The client withholds the body until `100 Continue`, which was never + // sent, so whether and when the body follows is unknown. + (req.expect_100_continue_pending_ && detail::has_framed_body(req))) { res.set_header("Connection", "close"); } else { std::string s = "timeout="; @@ -8653,9 +8732,10 @@ Server::write_content_with_provider(Stream &strm, const Request &req, } } else { if (res.is_chunked_content_provider_) { - auto type = detail::encoding_type(req, res); - - auto compressor = detail::make_compressor(type); + // Use the coding `apply_ranges()` chose when it wrote the headers; + // re-negotiating here would disagree with them, e.g. once a handler's + // own Content-Encoding header suppresses the negotiation. + auto compressor = detail::make_compressor(res.content_coding_); if (!compressor) { compressor = detail::make_unique<detail::nocompressor>(); } @@ -8812,6 +8892,13 @@ bool Server::read_content_core( } #endif + // The client is waiting for this before it sends the body. + if (req.expect_100_continue_pending_) { + req.expect_100_continue_pending_ = false; + detail::write_response_line(strm, StatusCode::Continue_100); + strm.write("\r\n"); + } + if (!detail::read_content(strm, req, payload_max_length_, res.status, nullptr, out, true)) { return false; @@ -8881,7 +8968,8 @@ bool Server::handle_file_request(Request &req, Response &res) { auto encoding = detail::EncodingType::None; if (static_file_compression_) { content_type = content_type_of(); - encoding = static_file_encoding(req, content_type, stat.size()); + encoding = + static_file_encoding(req, res, content_type, stat.size()); } // The ETag names the representation actually sent, so a client that @@ -9296,8 +9384,10 @@ bool Server::dispatch_request(Request &req, Response &res, // the ETag, which has to name the representation actually sent, and // `apply_static_file_compression()` go through this, so the two cannot drift // apart. -detail::EncodingType Server::static_file_encoding( - const Request &req, const std::string &content_type, size_t length) const { +detail::EncodingType +Server::static_file_encoding(const Request &req, const Response &res, + const std::string &content_type, + size_t length) const { if (!static_file_compression_) { return detail::EncodingType::None; } // Nothing to compress, and an empty file already answers with @@ -9322,14 +9412,14 @@ detail::EncodingType Server::static_file_encoding( return detail::EncodingType::None; } - return detail::encoding_type(req, content_type); + return detail::encoding_type(req, res, content_type); } // Compresses a file-backed content provider into `res.body` and takes over the // framing headers. Returns false when the response is left untouched. bool Server::apply_static_file_compression(const Request &req, Response &res) const { - auto type = res.file_content_encoding_; + auto type = res.content_coding_; if (type == detail::EncodingType::None || !res.content_provider_) { return false; } @@ -9353,7 +9443,7 @@ bool Server::apply_static_file_compression(const Request &req, res.content_provider_success_ = true; res.content_provider_ = nullptr; res.content_length_ = 0; - res.file_content_encoding_ = detail::EncodingType::None; + res.content_coding_ = detail::EncodingType::None; res.set_header("Content-Encoding", detail::encoding_name(type)); res.set_header("Vary", "Accept-Encoding"); @@ -9412,6 +9502,7 @@ void Server::apply_ranges(const Request &req, Response &res, if (res.content_provider_) { if (res.is_chunked_content_provider_) { res.set_header("Transfer-Encoding", "chunked"); + res.content_coding_ = type; if (type != detail::EncodingType::None) { res.set_header("Content-Encoding", detail::encoding_name(type)); res.set_header("Vary", "Accept-Encoding"); @@ -9568,8 +9659,8 @@ Server::process_request(Stream &strm, const std::string &remote_addr, // coding is not chunked, which leaves the body length undeterminable. The // latter must not fall through to the "no body" path, or the body bytes are // parsed as the next request on a persistent connection. - if (req.has_header("Transfer-Encoding") && - (req.get_header_value_u64("Content-Length") > 0 || + if (detail::has_conflicting_content_length(req.headers) || + (req.has_header("Transfer-Encoding") && !detail::is_chunked_transfer_encoding(req.headers))) { connection_closed = true; res.status = StatusCode::BadRequest_400; @@ -9645,19 +9736,20 @@ Server::process_request(Stream &strm, const std::string &remote_addr, // case-insensitive, and a 100-continue expectation in an HTTP/1.0 request // must be ignored. An expectation we do not recognize is left alone; the // 417 the section allows for one is a MAY, not a requirement. + // + // `100 Continue` itself is deferred until the body is actually read (see + // read_content_core), so a request rejected by a later handler never + // invites the client to send a body nobody will read. if (req.version != "HTTP/1.0" && detail::has_header_token(req.headers, "Expect", "100-continue")) { int status = StatusCode::Continue_100; if (expect_100_continue_handler_) { status = expect_100_continue_handler_(req, res); } - switch (status) { - case StatusCode::Continue_100: - case StatusCode::ExpectationFailed_417: - detail::write_response_line(strm, status); - strm.write("\r\n"); - break; - default: + if (status == StatusCode::Continue_100) { + req.expect_100_continue_pending_ = true; + } else { + if (res.status == -1) { res.status = status; } connection_closed = true; return write_response(strm, true, req, res); } @@ -9670,18 +9762,25 @@ Server::process_request(Stream &strm, const std::string &remote_addr, }; // WebSocket upgrade - // Check pre_routing_handler_ before upgrading so that authentication - // and other middleware can reject the request with an HTTP response - // (e.g., 401) before the protocol switches. + // Run pre_routing_handler_ and pre_request_handler_ before upgrading so + // that authentication and other middleware can reject the request with an + // HTTP response (e.g., 401) before the protocol switches. if (detail::is_websocket_upgrade(req)) { if (pre_routing_handler_ && pre_routing_handler_(req, res) == HandlerResponse::Handled) { if (res.status == -1) { res.status = StatusCode::OK_200; } - return write_response(strm, close_connection, req, res); + return write_response_with_content(strm, close_connection, req, res); } // Find matching WebSocket handler for (const auto &entry : websocket_handlers_) { if (entry.matcher->match(req)) { + req.matched_route = entry.matcher->pattern(); + if (pre_request_handler_ && + pre_request_handler_(req, res) == HandlerResponse::Handled) { + if (res.status == -1) { res.status = StatusCode::OK_200; } + return write_response_with_content(strm, close_connection, req, res); + } + // Compute accept key auto client_key = req.get_header_value("Sec-WebSocket-Key"); auto accept_key = detail::websocket_accept_key(client_key); @@ -9734,7 +9833,7 @@ Server::process_request(Stream &strm, const std::string &remote_addr, auto ws_strm = std::unique_ptr<Stream>(new detail::WebSocketSSLStream( strm.socket(), const_cast<tls::session_t>(req.ssl), - CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND, 0, + CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND, 0, write_timeout_sec_, write_timeout_usec_)); ws::WebSocket ws(std::move(ws_strm), req, true, websocket_ping_interval_sec_, @@ -9744,7 +9843,8 @@ Server::process_request(Stream &strm, const std::string &remote_addr, } #endif // Use WebSocket-specific read timeout instead of HTTP timeout - strm.set_read_timeout(CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND, 0); + strm.set_read_timeout(CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND, + 0); ws::WebSocket ws(strm, req, true, websocket_ping_interval_sec_, websocket_max_missed_pongs_); entry.handler(req, ws); @@ -9808,7 +9908,7 @@ Server::process_request(Stream &strm, const std::string &remote_addr, detail::set_file_content_provider( res, mm, content_type, - static_file_encoding(req, content_type, mm->size())); + static_file_encoding(req, res, content_type, mm->size())); } } @@ -10228,8 +10328,12 @@ Result ClientImpl::send_(Request &&req) { void ClientImpl::prepare_default_headers(Request &r, bool for_stream, const std::string &ct) { (void)for_stream; - for (const auto &header : default_headers_) { - if (!r.has_header(header.first)) { r.headers.insert(header); } + // Default headers are meant for the origin and may carry its credentials, so + // keep them off the CONNECT request the proxy reads. + if (r.method != "CONNECT") { + for (const auto &header : default_headers_) { + if (!r.has_header(header.first)) { r.headers.insert(header); } + } } // RFC 9110 5.3 recommends sending control data such as Host first, so @@ -10379,6 +10483,17 @@ ClientImpl::open_stream(const std::string &method, const std::string &path, return handle; } + // Same framing check as ClientImpl::process_request(). A HEAD or bodyless + // (204/304) response legitimately carries framing headers with no body. + if (method != "HEAD" && + handle.response->status != StatusCode::NoContent_204 && + handle.response->status != StatusCode::NotModified_304 && + detail::has_conflicting_content_length(handle.response->headers)) { + handle.error = Error::Read; + handle.response.reset(); + return handle; + } + handle.body_reader_.stream = handle.stream_; handle.body_reader_.payload_max_length = payload_max_length_; @@ -10525,22 +10640,45 @@ ssize_t ChunkedDecoder::read_payload(char *buf, size_t len, stream_line_reader lr(strm, line_buf, sizeof(line_buf)); if (!lr.getline()) { return -1; } + // Everything below is bounded by eol rather than by the buffer's NUL, so + // the line terminator is never mistaken for line content. + const char *eol = lr.ptr() + lr.size(); + if (lr.end_with_crlf()) { + eol -= 2; + } else if (eol != lr.ptr() && eol[-1] == '\n') { + // Only reachable under CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR, where + // getline() ends the line on a bare LF. That LF is the terminator, so it + // has to come off here or the check below would reject the line. + eol -= 1; + } + // RFC 9112 §7.1: chunk-size = 1*HEXDIG const char *p = lr.ptr(); int v = 0; - if (!is_hex(*p, v)) { return -1; } + if (p == eol || !is_hex(*p, v)) { return -1; } size_t chunk_len = 0; constexpr size_t chunk_len_max = (std::numeric_limits<size_t>::max)(); - for (; is_hex(*p, v); ++p) { + for (; p < eol && is_hex(*p, v); ++p) { if (chunk_len > (chunk_len_max >> 4)) { return -1; } chunk_len = (chunk_len << 4) | static_cast<size_t>(v); } - while (is_space_or_tab(*p)) { + while (p < eol && is_space_or_tab(*p)) { ++p; } - if (*p != '\0' && *p != ';' && *p != '\r' && *p != '\n') { return -1; } + + // RFC 9112 §7.1.1: only a chunk-ext may sit between the size and the line + // terminator, and it is built from tokens and quoted-strings, so it never + // holds a CR, LF or any other control character. getline() reads up to the + // CRLF, so a bare LF left in here would be swallowed as extension text + // while an intermediary that ends the line on it delimits the chunks + // differently, and the two disagree on where the body ends (request + // smuggling). + if (p < eol && *p != ';') { return -1; } + for (; p < eol; ++p) { + if (!is_space_or_tab(*p) && !fields::is_field_vchar(*p)) { return -1; } + } if (chunk_len == 0) { chunk_remaining = 0; @@ -10910,24 +11048,24 @@ bool ClientImpl::write_request(Stream &strm, Request &req, } } - if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) { - if (!req.has_header("Authorization")) { + // A CONNECT request is read by the proxy; everything sent through the tunnel + // it opens is read by the origin. Each credential goes only to its own hop. + auto is_connect = req.method == "CONNECT"; + + if (!is_connect && !req.has_header("Authorization")) { + if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) { req.headers.insert(make_basic_authentication_header( basic_auth_username_, basic_auth_password_, false)); - } - } - - if (!bearer_token_auth_token_.empty()) { - if (!req.has_header("Authorization")) { + } else if (!bearer_token_auth_token_.empty()) { req.headers.insert(make_bearer_token_authentication_header( bearer_token_auth_token_, false)); } } - // Proxy-Authorization is only sent when the proxy is actually used for - // this target — otherwise NO_PROXY-matched requests would leak proxy - // credentials directly to the destination server. - if (is_proxy_enabled_for_host(host_)) { + // Proxy-Authorization is only sent when the proxy reads this message — + // otherwise NO_PROXY-matched requests, and requests inside a TLS tunnel, + // would leak proxy credentials to the destination server. + if (is_proxy_enabled_for_host(host_) && (!is_ssl() || is_connect)) { if (!proxy_basic_auth_username_.empty() && !proxy_basic_auth_password_.empty() && !req.has_header("Proxy-Authorization")) { @@ -10969,9 +11107,10 @@ bool ClientImpl::write_request(Stream &strm, Request &req, // Write request line and headers if (detail::write_request_line(bstrm, req.method, path_with_query) < 0) { - // A rejected target (e.g. CR/LF smuggled in via a decoded redirect - // Location under set_path_encode(false)) must fail the request cleanly - // instead of emitting a request-line-less, header-injecting request. + // A rejected method (not a token, e.g. carrying CR/LF) or target (e.g. + // CR/LF smuggled in via a decoded redirect Location under + // set_path_encode(false)) must fail the request cleanly instead of + // emitting a request-line-less, header-injecting request. error = Error::Write; output_error_log(error, &req); return false; @@ -11323,6 +11462,17 @@ bool ClientImpl::process_request(Stream &strm, Request &req, // Body if ((res.status != StatusCode::NoContent_204) && req.method != "HEAD" && req.method != "CONNECT") { + // Reject ambiguous framing (RFC 9112 §6.3). Unlike a request, a response + // whose final transfer coding is not chunked is not ambiguous: its body + // runs until the server closes the connection, so it is not rejected. + // HEAD/204 are excluded above and a 304 carries no body. + if (res.status != StatusCode::NotModified_304 && + detail::has_conflicting_content_length(res.headers)) { + error = Error::Read; + output_error_log(error, &req); + return false; + } + auto redirect = 300 < res.status && res.status < 400 && res.status != StatusCode::NotModified_304 && follow_location_; @@ -14554,11 +14704,11 @@ void shutdown(session_t session, bool graceful) { auto ssl = static_cast<SSL *>(session); if (graceful) { - // First call sends close_notify - if (SSL_shutdown(ssl) == 0) { - // Second call waits for peer's close_notify - SSL_shutdown(ssl); - } + // Send close_notify without waiting for the peer's. The connection is + // closed right after this, so a unidirectional shutdown is enough, and an + // idle peer that never answers would otherwise hold this thread until the + // read timeout. The other backends do not wait either. + SSL_shutdown(ssl); } } @@ -17562,8 +17712,16 @@ ReadResult WebSocket::read(std::string &msg) { std::string payload; bool fin; - if (!impl::read_websocket_frame(strm_, opcode, payload, fin, is_server_, - CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) { + impl::FrameRead r = + impl::read_websocket_frame(strm_, opcode, payload, fin, is_server_, + CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH); + // A timeout landed on a frame boundary: the connection is untouched and + // still usable, so hand control back without closing it. That is only + // useful to a caller who asked for the timeout; the compile-time default + // is a backstop against a peer gone quiet, and elapsing it closes the + // connection so a plain `while (ws.read(msg))` loop ends. + if (r == impl::FrameRead::Timeout && read_timeout_set_) { return Timeout; } + if (r != impl::FrameRead::Ok) { closed_ = true; return Fail; } @@ -17600,9 +17758,14 @@ ReadResult WebSocket::read(std::string &msg) { Opcode cont_opcode; std::string cont_payload; bool cont_fin; - if (!impl::read_websocket_frame( + // A timeout is not reportable here: half of a fragmented message is + // already in `msg` and read() has no way to resume it, so it is a + // failure like any other. Timeouts are only ever seen on a message + // boundary. + if (impl::read_websocket_frame( strm_, cont_opcode, cont_payload, cont_fin, is_server_, - CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) { + CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH) != + impl::FrameRead::Ok) { closed_ = true; return Fail; } @@ -17696,7 +17859,8 @@ void WebSocket::close(CloseStatus status, const std::string &reason) { Opcode op; std::string resp; bool fin; - while (impl::read_websocket_frame(strm_, op, resp, fin, is_server_, 125)) { + while (impl::read_websocket_frame(strm_, op, resp, fin, is_server_, 125) == + impl::FrameRead::Ok) { if (op == Opcode::Close) { break; } } } @@ -17741,6 +17905,15 @@ const Request &WebSocket::request() const { return req_; } bool WebSocket::is_open() const { return !closed_; } +void WebSocket::set_read_timeout(time_t sec, time_t usec) { + // 0 waits forever here, as it does for SO_RCVTIMEO. The stream waits with + // poll(), where 0 would instead mean "return immediately", so hand it the + // negative poll uses for an unbounded wait. + if (sec == 0 && usec == 0) { sec = -1; } + strm_.set_read_timeout(sec, usec); + read_timeout_set_ = true; +} + // WebSocketClient implementation WebSocketClient::WebSocketClient( const std::string &scheme_host_port_path, const Headers &headers) @@ -17843,6 +18016,16 @@ void WebSocketClient::shutdown_and_close() { bool WebSocketClient::create_stream(std::unique_ptr<Stream> &strm, Error &error, int &ssl_error, uint64_t &ssl_backend_error) { + // A read timeout of 0 means "wait forever", the way SO_RCVTIMEO reads it. + // The streams wait with poll(), where 0 instead means "return immediately", + // so they are given the negative poll uses for an unbounded wait. + auto unbounded = read_timeout_sec_ == 0 && read_timeout_usec_ == 0; + time_t strm_read_sec = unbounded ? -1 : read_timeout_sec_; + time_t strm_read_usec = unbounded ? 0 : read_timeout_usec_; + // The handshake belongs to establishing the connection, so an unset read + // timeout leaves it bounded by the connection timeout instead of forever. + time_t hs_sec = unbounded ? connection_timeout_sec_ : read_timeout_sec_; + time_t hs_usec = unbounded ? connection_timeout_usec_ : read_timeout_usec_; #ifdef CPPHTTPLIB_SSL_ENABLED if (is_ssl_) { // A plain flag rather than SSLClient::load_certs()'s call_once: connect() @@ -17862,8 +18045,8 @@ bool WebSocketClient::create_stream(std::unique_ptr<Stream> &strm, detail::ClientTlsSessionError tls_error; if (!detail::setup_client_tls_session(host_, tls_ctx_, tls_session_, sock_, server_certificate_verification_, - read_timeout_sec_, read_timeout_usec_, - &tls_error, options)) { + hs_sec, hs_usec, &tls_error, + options)) { error = tls_error.error; ssl_error = tls_error.ssl_error; ssl_backend_error = tls_error.backend_error; @@ -17871,17 +18054,19 @@ bool WebSocketClient::create_stream(std::unique_ptr<Stream> &strm, } strm = std::unique_ptr<Stream>(new detail::WebSocketSSLStream( - sock_, tls_session_, read_timeout_sec_, read_timeout_usec_, - write_timeout_sec_, write_timeout_usec_)); + sock_, tls_session_, strm_read_sec, strm_read_usec, write_timeout_sec_, + write_timeout_usec_)); return true; } #else (void)error; (void)ssl_error; (void)ssl_backend_error; + (void)hs_sec; + (void)hs_usec; #endif strm = std::unique_ptr<Stream>( - new detail::SocketStream(sock_, read_timeout_sec_, read_timeout_usec_, + new detail::SocketStream(sock_, strm_read_sec, strm_read_usec, write_timeout_sec_, write_timeout_usec_)); return true; } @@ -17951,6 +18136,9 @@ Result WebSocketClient::connect() { ws_ = std::unique_ptr<WebSocket>(new WebSocket(std::move(strm), req, false, websocket_ping_interval_sec_, websocket_max_missed_pongs_)); + // The stream was created with the timeout already; tell the WebSocket + // whether it came from the caller, so read() knows to report it as Timeout. + ws_->read_timeout_set_ = read_timeout_set_; return Result{Error::Success, upgrade.status, std::move(upgrade.headers)}; } @@ -17983,6 +18171,10 @@ const std::string &WebSocketClient::subprotocol() const { void WebSocketClient::set_read_timeout(time_t sec, time_t usec) { read_timeout_sec_ = sec; read_timeout_usec_ = usec; + read_timeout_set_ = true; + // The members above only seed the next connect(); read() consults the + // stream, so an already-open connection has to be told directly. + if (ws_) { ws_->set_read_timeout(sec, usec); } } void WebSocketClient::set_write_timeout(time_t sec, time_t usec) { diff --git a/vendor/cpp-httplib/httplib.h b/vendor/cpp-httplib/httplib.h index ca7c96a41a38..ba603ed25b6f 100644 --- a/vendor/cpp-httplib/httplib.h +++ b/vendor/cpp-httplib/httplib.h @@ -8,8 +8,8 @@ #ifndef CPPHTTPLIB_HTTPLIB_H #define CPPHTTPLIB_HTTPLIB_H -#define CPPHTTPLIB_VERSION "0.54.1" -#define CPPHTTPLIB_VERSION_NUM "0x003601" +#define CPPHTTPLIB_VERSION "0.57.1" +#define CPPHTTPLIB_VERSION_NUM "0x003901" #ifdef _WIN32 #if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00 @@ -215,8 +215,36 @@ #define CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH 16777216 #endif -#ifndef CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND -#define CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND 300 +// One macro used to set the read timeout for both sides. They want different +// defaults: a client's read timeout is the caller's own tool (it waits forever +// until asked not to), while a server keeps a ceiling that reclaims a worker +// from a peer that has gone quiet. The old name still works and sets both. +#ifdef CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#pragma message( \ + "CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND is deprecated; define " \ + "CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND and/or " \ + "CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND instead") +#ifndef CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND \ + CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#endif +#ifndef CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND \ + CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#endif +#endif + +// 0 waits forever. A read timeout is how a caller gets control back to send on +// the same connection; it is not a liveness check (that is ping/pong). Only a +// timeout set at runtime through set_read_timeout() is reported as +// ws::Timeout; when one of these compile-time defaults elapses, read() returns +// ws::Fail and closes the connection. +#ifndef CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND 0 +#endif + +#ifndef CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND 300 #endif #ifndef CPPHTTPLIB_WEBSOCKET_CLOSE_TIMEOUT_SECOND @@ -1728,6 +1756,7 @@ struct Request { // private members... bool body_consumed_ = false; + bool expect_100_continue_pending_ = false; size_t redirect_count_ = CPPHTTPLIB_REDIRECT_MAX_COUNT; size_t content_length_ = 0; ContentProvider content_provider_; @@ -1817,10 +1846,12 @@ struct Response { std::string file_content_path_; std::string file_content_content_type_; - // Content coding chosen for a file-backed content provider, decided once - // where the file is opened so that the ETag and the body cannot disagree. - // `EncodingType::None` for every other kind of response. - detail::EncodingType file_content_encoding_ = detail::EncodingType::None; + // Content coding chosen for the response body, decided once so that the + // headers and the body cannot disagree: where the file is opened for a + // file-backed content provider (keeping the ETag honest), and in + // `apply_ranges()` for a chunked content provider. `EncodingType::None` + // for every other kind of response. + detail::EncodingType content_coding_ = detail::EncodingType::None; }; enum class Error { @@ -2359,6 +2390,7 @@ class Server { bool parse_request_line(const char *s, Request &req) const; detail::EncodingType static_file_encoding(const Request &req, + const Response &res, const std::string &content_type, size_t length) const; bool apply_static_file_compression(const Request &req, Response &res) const; @@ -3663,6 +3695,9 @@ ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags); EncodingType encoding_type(const Request &req, const std::string &content_type); +EncodingType encoding_type(const Request &req, const Response &res, + const std::string &content_type); + EncodingType encoding_type(const Request &req, const Response &res); class BufferStream final : public Stream { @@ -4345,7 +4380,11 @@ enum class CloseStatus : uint16_t { InternalError = 1011, }; -enum ReadResult : int { Fail = 0, Text = 1, Binary = 2 }; +// Timeout is returned only when a read timeout was set and it elapsed before +// any byte of a frame arrived: nothing was consumed and the connection is +// still open, so the caller can send on it and read again. `msg` is left +// untouched, so a `while (ws.read(msg))` loop must not treat it as a message. +enum ReadResult : int { Fail = 0, Text = 1, Binary = 2, Timeout = 3 }; // Result of WebSocketClient::connect(). Truthy only when the WebSocket // upgrade handshake fully succeeded. On failure error() identifies the @@ -4405,6 +4444,18 @@ class WebSocket { const Request &request() const; bool is_open() const; + // Bound how long read() waits before returning Timeout. 0 waits forever. + // A server handler owns its connection's timeout this way; a client sets it + // through WebSocketClient. Safe to call while another thread is in read(). + // + // Only a timeout set here is reported as Timeout. The compile-time default + // (CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND) is a backstop rather + // than a request for control, so when it elapses read() returns Fail and + // closes the connection, and `while (ws.read(msg))` ends as it always has. + void set_read_timeout(time_t sec, time_t usec = 0); + template <class Rep, class Period> + void set_read_timeout(const std::chrono::duration<Rep, Period> &duration); + private: friend class httplib::Server; friend class WebSocketClient; @@ -4440,6 +4491,10 @@ class WebSocket { int max_missed_pongs_; int unacked_pings_ = 0; std::atomic<bool> closed_{false}; + // Set once the caller has bounded read() through set_read_timeout(). Until + // then the timeout in effect is the compile-time default, and elapsing it + // is a failure that closes the connection, not a Timeout. + std::atomic<bool> read_timeout_set_{false}; std::mutex write_mutex_; // Owned by whichever thread is parsing frames off strm_. Only one thread // may do so: read_websocket_frame() reads a payload until it has the whole @@ -4527,8 +4582,9 @@ class WebSocketClient { bool is_valid_ = false; socket_t sock_ = INVALID_SOCKET; std::unique_ptr<WebSocket> ws_; - time_t read_timeout_sec_ = CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND; + time_t read_timeout_sec_ = CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND; time_t read_timeout_usec_ = 0; + bool read_timeout_set_ = false; // see WebSocket::read_timeout_set_ time_t write_timeout_sec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND; time_t write_timeout_usec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND; time_t websocket_ping_interval_sec_ = @@ -4560,6 +4616,13 @@ class WebSocketClient { #endif }; +template <class Rep, class Period> +inline void WebSocket::set_read_timeout( + const std::chrono::duration<Rep, Period> &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); }); +} + template <class Rep, class Period> inline void WebSocketClient::set_read_timeout( const std::chrono::duration<Rep, Period> &duration) { @@ -4586,8 +4649,14 @@ namespace impl { bool is_valid_utf8(const std::string &s); -bool read_websocket_frame(Stream &strm, Opcode &opcode, std::string &payload, - bool &fin, bool expect_masked, size_t max_len); +// Three states, because a failure that consumed bytes and one that consumed +// none are not the same thing: the first has left the stream in the middle of +// a frame and the connection cannot be reused, the second can just be retried. +enum class FrameRead { Ok, Fail, Timeout }; + +FrameRead read_websocket_frame(Stream &strm, Opcode &opcode, + std::string &payload, bool &fin, + bool expect_masked, size_t max_len); } // namespace impl